From 24ab6cabb4534fbeacb0a3efe1def47d80d17983 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Tue, 8 Aug 2017 11:29:18 +0200 Subject: [PATCH 01/40] Added initial sieve metamodel --- intelmq/bots/experts/sieve/__init__.py | 0 intelmq/bots/experts/sieve/expert.py | 0 intelmq/bots/experts/sieve/filter.sieve | 53 ++++++++++++++++++ intelmq/bots/experts/sieve/sieve.tx | 73 +++++++++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 intelmq/bots/experts/sieve/__init__.py create mode 100644 intelmq/bots/experts/sieve/expert.py create mode 100644 intelmq/bots/experts/sieve/filter.sieve create mode 100644 intelmq/bots/experts/sieve/sieve.tx diff --git a/intelmq/bots/experts/sieve/__init__.py b/intelmq/bots/experts/sieve/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/bots/experts/sieve/filter.sieve b/intelmq/bots/experts/sieve/filter.sieve new file mode 100644 index 000000000..88a4f7d5c --- /dev/null +++ b/intelmq/bots/experts/sieve/filter.sieve @@ -0,0 +1,53 @@ + +if allof ('source.ip' = '192.168.13.13', 'feed.name' :contains 'cymru') { reject;keep } + +if source.ip == ['123.123.123.123', '234.234.234.234'] { reject } + +if source.ip == '123.123.123.123' || source.ip == '234.234.234.234' + +// string equality operator == +// string inequality operator != +// regex match =~ +// regex not match !~ + +// or operator: || +// and operator: && + +if source.ip ~= '[' + +if anumber == 3 { keep; drop; keep } // watch out for this + +if :exists source.asn { .... } +if :!exists source.asn { ... } + +if source.ip == '123.123.123.123' { + add source.asn = 'AS559'; // add key/value only if key not present yet + add! source.asn = 'AS559'; // force overwrite if exists + modify source.asn = 'AS559'; // edit key/value only if key is present + remove source.asn; +} + + + +event = { + source.ip = '123.123.123.123', + source.asn = 'AS559' +} + +event2 = { + source.ip = '234.234.234.234' +} + + +if source.asn == 'AS559' { // do not match if key doesn't exist + +} + +// simulate ':exists' operator +if source.asn =~ /.*/ { ... } // should only match if key 'source.asn' exists + +// simulate ':notexists' operator? +if source.asn !~ /.*/ { ... } // will not match because key does not exist + + + diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx new file mode 100644 index 000000000..6bc7d0c90 --- /dev/null +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -0,0 +1,73 @@ +SieveModel: rules*=Rule ; + +Rule: + 'if' condition=Condition '{' + actions*=Action[';'] + '}' +; + +// Condition: SingleCondition | ConditionList ; + +ConditionList: + combination=Combination '(' + conditions+=SingleCondition[','] + ')' +; + +Combination: + 'allof' // true if every single test in parenthesis are true (logical and) + | 'anyof' // true if at least one of the single tests in parenthesis is true (logical or) +; + +SingleCondition: key=STRING match=MatchRule ; + +MatchRule: StringMatchRule | NumericMatchRule ; + +StringMatchRule: match_type=StringMatchType value=Value; + +NumericMatchRule: match_type=NumericMatchType value=NUMBER; + +Value: SingleValue | ValueList ; + +SingleValue: value=STRING ; + +ValueList: '[' values+=SingleValue[','] ']' ; + +ExistMatchType: + ':exists', + | ':notexists' + +StringMatchType: + '==' // compares two whole strings with each other + | '!=' // test for string inequality + | ':contains' // sub-string match + | '=~' // match strings according to regular expression + | '!~' // inverse match with regular expression +; + +NumericMatchType: + '==' // equal + | '!=' // not equal + | '<' // less than + | '<=' // less than or equal + | '>' // greater than + | '>=' // greater than or equal + +; + +FilteringAction: + 'drop' | 'keep' +; + +ModificationAction: + 'add' key '=' value| +; + +Comment: + /(\/\/.*$)|(\/\*.*\*\/)/ // TODO: // line comments and /* block comments */, figure out the right regex + + /* bla + bla bla + bla + */ +; \ No newline at end of file From 6e6465de45e7638aa5b678c69f33200beca9e4af Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Tue, 8 Aug 2017 16:21:43 +0200 Subject: [PATCH 02/40] Refined sieve metamodel --- intelmq/bots/experts/sieve/sieve.tx | 71 ++++++++++++----------------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 6bc7d0c90..37e57d100 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -1,73 +1,62 @@ SieveModel: rules*=Rule ; Rule: - 'if' condition=Condition '{' + 'if' expr=Expression '{' actions*=Action[';'] '}' ; -// Condition: SingleCondition | ConditionList ; - -ConditionList: - combination=Combination '(' - conditions+=SingleCondition[','] - ')' -; - -Combination: - 'allof' // true if every single test in parenthesis are true (logical and) - | 'anyof' // true if at least one of the single tests in parenthesis is true (logical or) +Expression: conj=Conjunction ('||' conj=Conjunction)*; +Conjunction: cond=Condition ('&&' cond=Condition)*; +Condition: + match=StringMatch + | match=NumericMatch + | match=ExistMatch + | ( '(' match=Expression ')' ) ; -SingleCondition: key=STRING match=MatchRule ; - -MatchRule: StringMatchRule | NumericMatchRule ; - -StringMatchRule: match_type=StringMatchType value=Value; - -NumericMatchRule: match_type=NumericMatchType value=NUMBER; - -Value: SingleValue | ValueList ; - -SingleValue: value=STRING ; - -ValueList: '[' values+=SingleValue[','] ']' ; - -ExistMatchType: - ':exists', - | ':notexists' -StringMatchType: +StringMatch: key=Key op=StringOperator value=Value; +StringOperator: '==' // compares two whole strings with each other | '!=' // test for string inequality - | ':contains' // sub-string match + | ':contains' // sub-string match | '=~' // match strings according to regular expression | '!~' // inverse match with regular expression ; -NumericMatchType: +NumericMatch: key=Key op=NumericOperator value=NUMBER; +NumericOperator: '==' // equal | '!=' // not equal | '<' // less than | '<=' // less than or equal | '>' // greater than | '>=' // greater than or equal - ; +ExistMatch: op=ExistOperator key=STRING; +ExistOperator: ':exists' | ':notexists'; + +Key: /[a-z_.]+/; + +Value: SingleValue | ValueList ; +SingleValue: value=STRING ; +ValueList: '[' values+=SingleValue[','] ']' ; + + +Action: FilteringAction | AddAction | AddForceAction | ModifyAction | RemoveAction; + FilteringAction: 'drop' | 'keep' ; -ModificationAction: - 'add' key '=' value| -; + +AddAction: 'add' key=Key '=' value=STRING; +AddForceAction: 'add!' key=Key '=' value=STRING; +ModifyAction: 'modify' key=Key '=' value=STRING; +RemoveAction: 'remove' key=Key; Comment: /(\/\/.*$)|(\/\*.*\*\/)/ // TODO: // line comments and /* block comments */, figure out the right regex - - /* bla - bla bla - bla - */ ; \ No newline at end of file From 0e43dd80ed9cd617acdec5aaee825766f7d14e93 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Wed, 9 Aug 2017 11:43:32 +0200 Subject: [PATCH 03/40] Added first draft of sieve/expert.py with unittests (not functional) --- intelmq/bots/experts/sieve/expert.py | 54 +++++++++++++++++++ intelmq/tests/bots/experts/sieve/__init__.py | 0 intelmq/tests/bots/experts/sieve/test.sieve | 3 ++ .../tests/bots/experts/sieve/test_expert.py | 40 ++++++++++++++ setup.py | 1 + 5 files changed, 98 insertions(+) create mode 100644 intelmq/tests/bots/experts/sieve/__init__.py create mode 100644 intelmq/tests/bots/experts/sieve/test.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_expert.py diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index e69de29bb..7ce33fadc 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" +SieveExpertBot filters and modifies events based on a specification language similar to mail sieve. + +TODO: Document possible necessary configurations. +Parameters: + file: string +""" +from __future__ import unicode_literals + +# imports for additional libraries and intelmq +import os +import intelmq.lib.exceptions as exceptions +from intelmq.lib.bot import Bot +from intelmq.lib.utils import load_configuration +from textx.metamodel import metamodel_from_file +from textx.exceptions import TextXError + + +class SieveExpertBot(Bot): + + def init(self): + # read the sieve grammar + try: + filename = os.path.join(os.path.dirname(__file__), 'sieve.tx') + self.metamodel = metamodel_from_file(filename) + except TextXError as e: + self.logger.error('Could not process sieve grammar file.') + self.logger.error(e.message) + self.stop() + + # validate parameters + if not os.path.exists(self.parameters.file): + raise exceptions.InvalidArgument('file', got=self.parameters.file, expected='existing file') + + # parse sieve file + try: + self.metamodel.model_from_file(self.parameters.file) + except TextXError as e: + self.logger.error('Could not parse sieve file \'%r\'', self.parameters.file) + self.logger.error(e.message) + self.stop() + + + def process(self): + event = self.receive_message() + + # implement the logic here + + self.send_message(event) + self.acknowledge_message() + + +BOT = SieveExpertBot diff --git a/intelmq/tests/bots/experts/sieve/__init__.py b/intelmq/tests/bots/experts/sieve/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/intelmq/tests/bots/experts/sieve/test.sieve b/intelmq/tests/bots/experts/sieve/test.sieve new file mode 100644 index 000000000..b17fb5d8e --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test.sieve @@ -0,0 +1,3 @@ +if (source.ip == '127.0.0.1') { + keep +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py new file mode 100644 index 000000000..9fe2c7539 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import unittest +import os +import intelmq.lib.test as test +from intelmq.bots.experts.sieve.expert import SieveExpertBot + + +EXAMPLE_INPUT = {"__type": "Event", + "source.ip": "127.0.0.1", + "source.abuse_contact": "abuse@example.com", + "time.observation": "2017-01-01T00:00:00+00:00", + } + +class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for SieveExpertBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = SieveExpertBot + cls.default_input_message = EXAMPLE_INPUT + cls.sysconfig = {'file': os.path.join(os.path.dirname(__file__), 'test.sieve')} + + # This is an example how to test the log output +# def test_log_test_line(self): +# """ Test if bot does log example message. """ +# self.run_bot() +# self.assertRegexpMatches(self.loglines_buffer, "INFO - Lorem ipsum dolor sit amet") + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() +# self.assertMessageEqual(0, EXAMPLE_REPORT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/setup.py b/setup.py index b7262ef84..f40057a01 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ 'python-termstyle>=0.1.10', 'pytz>=2014.1', 'redis>=2.10.3', + 'requests>=2.2.0' ] if sys.version_info < (3, 5): REQUIRES.append('typing') From a187b3f19a1c818fe1063faab257948995140539 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Wed, 9 Aug 2017 15:37:30 +0200 Subject: [PATCH 04/40] Implemented parsing the sieve file (partially) --- intelmq/bots/experts/sieve/expert.py | 133 ++++++++++++++++-- intelmq/bots/experts/sieve/sieve.tx | 34 +++-- intelmq/tests/bots/experts/sieve/test.sieve | 5 +- .../tests/bots/experts/sieve/test_expert.py | 10 +- 4 files changed, 155 insertions(+), 27 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 7ce33fadc..2ecb67577 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -11,8 +11,8 @@ # imports for additional libraries and intelmq import os import intelmq.lib.exceptions as exceptions +import re from intelmq.lib.bot import Bot -from intelmq.lib.utils import load_configuration from textx.metamodel import metamodel_from_file from textx.exceptions import TextXError @@ -25,8 +25,8 @@ def init(self): filename = os.path.join(os.path.dirname(__file__), 'sieve.tx') self.metamodel = metamodel_from_file(filename) except TextXError as e: - self.logger.error('Could not process sieve grammar file.') - self.logger.error(e.message) + self.logger.error('Could not process sieve grammar file. Error in (%d, %d)', e.line, e.col) + self.logger.error(str(e)) # TODO: output textx exception message properly self.stop() # validate parameters @@ -35,20 +35,135 @@ def init(self): # parse sieve file try: - self.metamodel.model_from_file(self.parameters.file) + self.sieve = self.metamodel.model_from_file(self.parameters.file) except TextXError as e: - self.logger.error('Could not parse sieve file \'%r\'', self.parameters.file) - self.logger.error(e.message) + self.logger.error('Could not parse sieve file \'%r\', error in (%d, %d)', self.parameters.file, e.line, e.col) + self.logger.error(str(e)) # TODO: output textx exception message properly self.stop() - def process(self): event = self.receive_message() - # implement the logic here + keep = False + for rule in self.sieve.rules: + keep = SieveExpertBot.process_rule(rule, event) + if not keep: + break + + if keep: + self.send_message(event) - self.send_message(event) self.acknowledge_message() + @staticmethod + def process_rule(rule, event): + print("process rule") + match = SieveExpertBot.match_expression(rule.expr, event) + keep = True + if match: + for action in rule.actions: + keep = SieveExpertBot.process_action(action.action, event) + if not keep: + break + return keep + + @staticmethod + def match_expression(expr, event): + print("match_expression") + for conj in expr.conj: + if SieveExpertBot.process_conjunction(conj, event): + return True + return False + + @staticmethod + def process_conjunction(conj, event): + print("process_conjunction") + for cond in conj.cond: + if not SieveExpertBot.process_condition(cond, event): + return False + return True + + @staticmethod + def process_condition(cond, event): + print("process_condition") + match = cond.match + if match.__class__.__name__ == 'ExistMatch': + return SieveExpertBot.process_exist_match(match.key, event) + elif match.__class__.__name__ == 'StringMatch': + return SieveExpertBot.process_string_match(match.key, match.op, match.value, event) + elif match.__class__.__name__ == 'NumericMatch': + return SieveExpertBot.process_numeric_match(match.key, match.op, match.value, event) + elif match.__class__.__name__ == 'Expression': + return SieveExpertBot.match_expression(match, event) + pass + + @staticmethod + def process_exist_match(key, event): + return key in event + + @staticmethod + def process_string_match(key, op, value, event): + print("process_string_match") + if key not in event: + return False + + if value.__class__.__name__ == 'SingleStringValue': + return SieveExpertBot.process_string_operator(event[key], op, value.value) + elif value.__class__.__name__ == 'StringValueList': + for val in value.values: + if SieveExpertBot.process_string_operator(event[key], op, val): + return True + return False + + @staticmethod + def process_string_operator(lhs, op, rhs): + print("process_string_operator: %s %s %s" % (lhs, op, rhs)) + if op == '==': + print(lhs==rhs) + return lhs == rhs + elif op == '!=': + return lhs != rhs + elif op == ':contains': + return lhs.find(rhs) >= 0 + elif op == '=~': + return re.fullmatch(rhs, lhs) is not None + elif op == '!~': + return re.fullmatch(rhs, lhs) is None + + @staticmethod + def process_numeric_match(key, op, value, event): + if key not in event: + return False + + if value.__class__.__name__ == 'SingleNumericValue': + return SieveExpertBot.process_numeric_operator(event[key], op, value.value) + elif value.__class__.__name__ == 'NumericValueList': + for val in value.values: + if SieveExpertBot.process_numeric_operator(event[key], op, val): + return True + return False + + @staticmethod + def process_numeric_operator(lhs, op, rhs): + return eval(lhs + op + rhs) + + @staticmethod + def process_action(action, event): + print("process action") + if action.__class__.__name__ == 'DropAction': + return False + elif action.__class__.__name__ == 'AddAction': + if action.key not in event: + event[action.key] = action.value + elif action.__class__.__name__ == 'AddForceAction': + event[action.key] = action.value + elif action.__class__.__name__ == 'ModifyAction': + if action.key in event: + event[action.key] = action.value + elif action.__class__.__name__ == 'RemoveAction': + if action.key in event: + del event[action.key] + return True + BOT = SieveExpertBot diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 37e57d100..62b647f5d 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -1,3 +1,8 @@ +/* + Metamodel (grammar) for the SieveExpertBot file format. + For details, see: https://github.com/igordejanovic/textX +*/ + SieveModel: rules*=Rule ; Rule: @@ -16,7 +21,7 @@ Condition: ; -StringMatch: key=Key op=StringOperator value=Value; +StringMatch: key=Key op=StringOperator value=StringValue; StringOperator: '==' // compares two whole strings with each other | '!=' // test for string inequality @@ -25,7 +30,7 @@ StringOperator: | '!~' // inverse match with regular expression ; -NumericMatch: key=Key op=NumericOperator value=NUMBER; +NumericMatch: key=Key op=NumericOperator value=NumericValue; NumericOperator: '==' // equal | '!=' // not equal @@ -35,26 +40,25 @@ NumericOperator: | '>=' // greater than or equal ; -ExistMatch: op=ExistOperator key=STRING; +ExistMatch: op=ExistOperator key=Key; ExistOperator: ':exists' | ':notexists'; Key: /[a-z_.]+/; -Value: SingleValue | ValueList ; -SingleValue: value=STRING ; -ValueList: '[' values+=SingleValue[','] ']' ; - +StringValue: SingleStringValue | StringValueList ; +SingleStringValue: value=STRING ; +StringValueList: '[' values+=SingleStringValue[','] ']' ; -Action: FilteringAction | AddAction | AddForceAction | ModifyAction | RemoveAction; - -FilteringAction: - 'drop' | 'keep' -; +NumericValue: SingleNumericValue | NumericValueList ; +SingleNumericValue: value=NUMBER ; +NumericValueList: '[' values+=SingleNumericValue[','] ']' ; -AddAction: 'add' key=Key '=' value=STRING; -AddForceAction: 'add!' key=Key '=' value=STRING; -ModifyAction: 'modify' key=Key '=' value=STRING; +Action: action=DropAction | (action=AddAction | action=AddForceAction | action=ModifyAction | action=RemoveAction); +DropAction: 'drop'; +AddAction: 'add' key=Key '=' value=STRING; // add key/value without overwriting existing key +AddForceAction: 'add!' key=Key '=' value=STRING; // add key/value, overriding existing key +ModifyAction: 'modify' key=Key '=' value=STRING; // modify key/value, do not create if not exists RemoveAction: 'remove' key=Key; Comment: diff --git a/intelmq/tests/bots/experts/sieve/test.sieve b/intelmq/tests/bots/experts/sieve/test.sieve index b17fb5d8e..a59633494 100644 --- a/intelmq/tests/bots/experts/sieve/test.sieve +++ b/intelmq/tests/bots/experts/sieve/test.sieve @@ -1,3 +1,4 @@ if (source.ip == '127.0.0.1') { - keep -} \ No newline at end of file + add source.asn = '559' +} + diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 9fe2c7539..a142a6d27 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -13,6 +13,13 @@ "time.observation": "2017-01-01T00:00:00+00:00", } +EXAMPLE_OUTPUT = {"__type": "Event", + "source.ip": "127.0.0.1", + "source.abuse_contact": "abuse@example.com", + "time.observation": "2017-01-01T00:00:00+00:00", + "source.asn": 559 + } + class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for SieveExpertBot. @@ -33,7 +40,8 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ self.run_bot() -# self.assertMessageEqual(0, EXAMPLE_REPORT) + event = self.get_output_queue()[0] + self.assertMessageEqual(0, EXAMPLE_OUTPUT) if __name__ == '__main__': # pragma: no cover From 9eb33c6ae97b49de4dffae1d4e4df37715182199 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 9 Aug 2017 16:03:33 +0100 Subject: [PATCH 05/40] Sieve bot added to BOTS registration file --- intelmq/bots/BOTS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index 1db669791..2ab63a066 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -572,6 +572,13 @@ "module": "intelmq.bots.experts.taxonomy.expert", "parameters": {} }, + "Sievebot": { + "description": "Sievebot is the bot responsible to filter and modify intelMQ events.", + "module": "intelmq.bots.experts.sieve.expert", + "parameters": { + "file" :"/opt/intelmq/var/lib/bots/filter.sieve" + } + }, "Tor Nodes": { "description": "Tor Nodes is the bot responsible to check if an IP is an Tor Exit Node.", "module": "intelmq.bots.experts.tor_nodes.expert", From 54c05d0b2f9bcedd6005f4e795494df8b79ec853 Mon Sep 17 00:00:00 2001 From: Kevin Holvoet Date: Wed, 9 Aug 2017 16:51:21 +0200 Subject: [PATCH 06/40] Add comment regex in model --- intelmq/bots/experts/sieve/.gitignore | 2 + intelmq/bots/experts/sieve/README.md | 15 +++ intelmq/bots/experts/sieve/expert.py | 5 +- intelmq/bots/experts/sieve/sieve.tx | 6 +- intelmq/tests/bots/experts/sieve/test.sieve | 4 - .../tests/bots/experts/sieve/test_expert.py | 114 +++++++++++++++--- .../test_sieve_files/test_drop_event.sieve | 3 + .../test_sieve_files/test_keep_event.sieve | 3 + 8 files changed, 124 insertions(+), 28 deletions(-) create mode 100644 intelmq/bots/experts/sieve/.gitignore create mode 100644 intelmq/bots/experts/sieve/README.md delete mode 100644 intelmq/tests/bots/experts/sieve/test.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve diff --git a/intelmq/bots/experts/sieve/.gitignore b/intelmq/bots/experts/sieve/.gitignore new file mode 100644 index 000000000..3be615847 --- /dev/null +++ b/intelmq/bots/experts/sieve/.gitignore @@ -0,0 +1,2 @@ +*.dot + diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md new file mode 100644 index 000000000..6d7655fb1 --- /dev/null +++ b/intelmq/bots/experts/sieve/README.md @@ -0,0 +1,15 @@ +# Sieve Bot + +The sieve bot is used to filter and/or modify bots based on a set of rules. The +rules are specified in an external configuration file and with a syntax similar +to the [Sieve language](http://sieve.info/) used for mail filtering. + +Each rule defines a set of matching conditions on received events. Events can be +matched based on keys and +corresponding values + +## Examples +The following excerpts illustrate the + +## Parameters + * `file` - filesystem path the the sieve file diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 2ecb67577..b0355e68d 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -149,8 +149,9 @@ def process_numeric_operator(lhs, op, rhs): @staticmethod def process_action(action, event): - print("process action") - if action.__class__.__name__ == 'DropAction': + print("action " + action) + if action == 'drop': + print("drop that shit") return False elif action.__class__.__name__ == 'AddAction': if action.key not in event: diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 62b647f5d..42f4ccf7f 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -57,10 +57,8 @@ NumericValueList: '[' values+=SingleNumericValue[','] ']' ; Action: action=DropAction | (action=AddAction | action=AddForceAction | action=ModifyAction | action=RemoveAction); DropAction: 'drop'; AddAction: 'add' key=Key '=' value=STRING; // add key/value without overwriting existing key -AddForceAction: 'add!' key=Key '=' value=STRING; // add key/value, overriding existing key +AddForceAction: 'add!' key=Key '=' value=STRING; // add key/value, overwriting existing key ModifyAction: 'modify' key=Key '=' value=STRING; // modify key/value, do not create if not exists RemoveAction: 'remove' key=Key; -Comment: - /(\/\/.*$)|(\/\*.*\*\/)/ // TODO: // line comments and /* block comments */, figure out the right regex -; \ No newline at end of file +Comment: /\/\/.*$/ ; \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test.sieve b/intelmq/tests/bots/experts/sieve/test.sieve deleted file mode 100644 index a59633494..000000000 --- a/intelmq/tests/bots/experts/sieve/test.sieve +++ /dev/null @@ -1,4 +0,0 @@ -if (source.ip == '127.0.0.1') { - add source.asn = '559' -} - diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index a142a6d27..957583bc0 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -13,13 +13,6 @@ "time.observation": "2017-01-01T00:00:00+00:00", } -EXAMPLE_OUTPUT = {"__type": "Event", - "source.ip": "127.0.0.1", - "source.abuse_contact": "abuse@example.com", - "time.observation": "2017-01-01T00:00:00+00:00", - "source.asn": 559 - } - class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for SieveExpertBot. @@ -28,20 +21,105 @@ class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): cls.bot_reference = SieveExpertBot - cls.default_input_message = EXAMPLE_INPUT - cls.sysconfig = {'file': os.path.join(os.path.dirname(__file__), 'test.sieve')} + #cls.sysconfig = {'file': os.path.join(os.path.dirname(__file__), 'test.sieve')} + + def test_or_match(self): + """ Test Or Operator in match""" + + def test_and_match(self): + """ Test And Operator in match""" + + def test_precedence(self): + """ Test precedence of operators """ + + def test_string_equal_match(self): + """ Test == string match """ + + def test_string_not_equal_match(self): + """ Test != string match """ + + def test_string_contains_match(self): + """ Test :contains string match """ + + def test_string_regex_match(self): + """ Test =~ string match """ + + def test_string_inverse_regex_match(self): + """ Test !~ string match """ + + def test_numeric_equal_match(self): + """ Test == numeric match """ + + def test_numeric_not_equal_match(self): + """ Test != numeric match """ + + def test_numeric_less_than_match(self): + """ Test < numeric match """ + + def test_numeric_less_than_or_equal_match(self): + """ Test <= numeric match """ + + def test_numeric_greater_than_match(self): + """ Test > numeric match """ - # This is an example how to test the log output -# def test_log_test_line(self): -# """ Test if bot does log example message. """ -# self.run_bot() -# self.assertRegexpMatches(self.loglines_buffer, "INFO - Lorem ipsum dolor sit amet") + def test_numeric_greater_than_or_equal_match(self): + """ Test >= numeric match """ - def test_event(self): - """ Test if correct Event has been produced. """ + def test_exists_match(self): + """ Test :exists match """ + + def test_not_exists_match(self): + """ Test :notexists match """ + + def test_string_match_value_list(self): + """ Test string match with StringValueList """ + + def test_numeric_match_value_list(self): + """ Test numeric match with StringValueList """ + + def test_drop_event(self): + """ Test if matched event is dropped. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_drop_event.sieve') + + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = "deleteme" + self.input_message = event2 + self.run_bot() + self.assertOutputQueueLen(0) + + def test_keep_event(self): + """ Test if matched event is kept. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_keep_event.sieve') + + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = "keepme" + self.input_message = event2 self.run_bot() - event = self.get_output_queue()[0] - self.assertMessageEqual(0, EXAMPLE_OUTPUT) + self.assertMessageEqual(0, event2) + + def test_add(self): + """ Test adding key/value pairs """ + + def test_add_force(self): + """ Test adding key/value pairs, overwriting existing key """ + + def test_modify(self): + """ Test modifying key/value pairs """ + + def test_remove(self): + """ Test removing keys """ + + if __name__ == '__main__': # pragma: no cover diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve new file mode 100644 index 000000000..49921b68c --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve @@ -0,0 +1,3 @@ +if comment == 'deleteme' { + drop +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve new file mode 100644 index 000000000..9ce5be89e --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve @@ -0,0 +1,3 @@ +if comment == 'keepme' { + +} \ No newline at end of file From e3eaa46f3d63f0bef106f63107b2630c8ae2a352 Mon Sep 17 00:00:00 2001 From: Kevin Holvoet Date: Thu, 10 Aug 2017 17:06:57 +0200 Subject: [PATCH 07/40] Remove debugging prints --- intelmq/bots/experts/sieve/README.md | 30 ++- intelmq/bots/experts/sieve/expert.py | 18 +- intelmq/bots/experts/sieve/sieve.tx | 6 +- .../tests/bots/experts/sieve/test_expert.py | 195 +++++++++++++++++- .../test_sieve_files/test_and_match.sieve | 3 + .../test_multiple_actions.sieve | 5 + .../test_numeric_equal_match.sieve | 1 + .../test_numeric_greater_than_match.sieve | 1 + ..._numeric_greater_than_or_equal_match.sieve | 1 + .../test_numeric_less_than_match.sieve | 1 + ...est_numeric_less_than_or_equal_match.sieve | 1 + .../test_numeric_not_equal_match.sieve | 1 + .../test_sieve_files/test_or_match.sieve | 3 + 13 files changed, 246 insertions(+), 20 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_and_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index 6d7655fb1..7434d0152 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -5,11 +5,35 @@ rules are specified in an external configuration file and with a syntax similar to the [Sieve language](http://sieve.info/) used for mail filtering. Each rule defines a set of matching conditions on received events. Events can be -matched based on keys and -corresponding values +matched based on keys and values in the event. If the processed event matches a +rule's conditions, the corresponding actions are performed. Actions can specify +whether the event should be kept or dropped in the pipeline (filtering actions) +or if keys and values should be changed (modification actions). ## Examples -The following excerpts illustrate the +The following excerpts illustrate the basic features of the sieve file format. + +### Filtering based on event properties + +``` +if source.ip == '127.0.0.1' { + drop +} + +if :notexists source.abuse_contact || source.abuse_contact =~ '.*@example.com' { + drop +} +``` + +### Modification based on event properties + +``` +if classification.type == ['phishing', 'malware'] && source.fqdn =~ '.*\.(ch|li)$' { + add comment = 'domainabuse' + modify classification.taxonomy = 'fraud' + remove extra.comments +} +``` ## Parameters * `file` - filesystem path the the sieve file diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index b0355e68d..e9ccfdee5 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -57,7 +57,6 @@ def process(self): @staticmethod def process_rule(rule, event): - print("process rule") match = SieveExpertBot.match_expression(rule.expr, event) keep = True if match: @@ -69,7 +68,6 @@ def process_rule(rule, event): @staticmethod def match_expression(expr, event): - print("match_expression") for conj in expr.conj: if SieveExpertBot.process_conjunction(conj, event): return True @@ -77,7 +75,6 @@ def match_expression(expr, event): @staticmethod def process_conjunction(conj, event): - print("process_conjunction") for cond in conj.cond: if not SieveExpertBot.process_condition(cond, event): return False @@ -85,7 +82,6 @@ def process_conjunction(conj, event): @staticmethod def process_condition(cond, event): - print("process_condition") match = cond.match if match.__class__.__name__ == 'ExistMatch': return SieveExpertBot.process_exist_match(match.key, event) @@ -103,7 +99,6 @@ def process_exist_match(key, event): @staticmethod def process_string_match(key, op, value, event): - print("process_string_match") if key not in event: return False @@ -117,9 +112,7 @@ def process_string_match(key, op, value, event): @staticmethod def process_string_operator(lhs, op, rhs): - print("process_string_operator: %s %s %s" % (lhs, op, rhs)) if op == '==': - print(lhs==rhs) return lhs == rhs elif op == '!=': return lhs != rhs @@ -145,22 +138,21 @@ def process_numeric_match(key, op, value, event): @staticmethod def process_numeric_operator(lhs, op, rhs): - return eval(lhs + op + rhs) + return eval(str(lhs) + op + str(rhs)) @staticmethod def process_action(action, event): - print("action " + action) + print(type(event)) if action == 'drop': - print("drop that shit") return False elif action.__class__.__name__ == 'AddAction': if action.key not in event: - event[action.key] = action.value + event.add(action.key, action.value) elif action.__class__.__name__ == 'AddForceAction': - event[action.key] = action.value + event.add(action.key, action.value, overwrite=True) elif action.__class__.__name__ == 'ModifyAction': if action.key in event: - event[action.key] = action.value + event.change(action.key, action.value) elif action.__class__.__name__ == 'RemoveAction': if action.key in event: del event[action.key] diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 42f4ccf7f..36f525f89 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -7,7 +7,7 @@ SieveModel: rules*=Rule ; Rule: 'if' expr=Expression '{' - actions*=Action[';'] + actions*=Action '}' ; @@ -34,10 +34,10 @@ NumericMatch: key=Key op=NumericOperator value=NumericValue; NumericOperator: '==' // equal | '!=' // not equal - | '<' // less than | '<=' // less than or equal - | '>' // greater than | '>=' // greater than or equal + | '<' // less than + | '>' // greater than ; ExistMatch: op=ExistOperator key=Key; diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 957583bc0..042498457 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -21,13 +21,86 @@ class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): @classmethod def set_bot(cls): cls.bot_reference = SieveExpertBot - #cls.sysconfig = {'file': os.path.join(os.path.dirname(__file__), 'test.sieve')} def test_or_match(self): """ Test Or Operator in match""" + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_or_match.sieve') + + # Expressions: TRUE || TRUE => TRUE + truetrue = EXAMPLE_INPUT.copy() + truetrue['comment'] = "I am TRUE in OR clause" + truetrue_result = truetrue.copy() + truetrue_result['source.ip'] = "10.9.8.7" + self.input_message = truetrue + self.run_bot() + self.assertMessageEqual(0, truetrue_result) + + # Expressions: TRUE || FALSE => TRUE + truefalse = EXAMPLE_INPUT.copy() + truefalse['comment'] = "I am NOT True in OR clause" + truefalse_result = truefalse.copy() + truefalse_result['source.ip'] = "10.9.8.7" + self.input_message = truefalse + self.run_bot() + self.assertMessageEqual(0, truefalse_result) + + # Expressions: FALSE || TRUE => TRUE + falsetrue = EXAMPLE_INPUT.copy() + falsetrue['source.abuse_contact'] = "test@test.eu" + falsetrue['comment'] = "I am TRUE in OR clause" + falsetrue_result = falsetrue.copy() + falsetrue_result['source.ip'] = "10.9.8.7" + self.input_message = falsetrue + self.run_bot() + self.assertMessageEqual(0, falsetrue_result) + + # Expressions: FALSE || FALSE => FALSE + falsefalse = EXAMPLE_INPUT.copy() + falsefalse['source.abuse_contact'] = "test@test.eu" + falsefalse['comment'] = "I am NOT True in OR clause" + falsefalse_result = falsefalse.copy() + self.input_message = falsefalse + self.run_bot() + self.assertMessageEqual(0, falsefalse_result) def test_and_match(self): """ Test And Operator in match""" + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_and_match.sieve') + + # Expressions: TRUE && TRUE => TRUE + truetrue = EXAMPLE_INPUT.copy() + truetrue['comment'] = "I am TRUE in AND clause" + truetrue_result = truetrue.copy() + truetrue_result['source.ip'] = "10.9.8.7" + self.input_message = truetrue + self.run_bot() + self.assertMessageEqual(0, truetrue_result) + + # Expressions: TRUE && FALSE => FALSE + truefalse = EXAMPLE_INPUT.copy() + truefalse['comment'] = "I am NOT True in AND clause" + truefalse_result = truefalse.copy() + self.input_message = truefalse + self.run_bot() + self.assertMessageEqual(0, truefalse_result) + + # Expressions: FALSE && TRUE => FALSE + falsetrue = EXAMPLE_INPUT.copy() + falsetrue['source.abuse_contact'] = "test@test.eu" + falsetrue['comment'] = "I am TRUE in AND clause" + falsetrue_result = falsetrue.copy() + self.input_message = falsetrue + self.run_bot() + self.assertMessageEqual(0, falsetrue_result) + + # Expressions: FALSE && FALSE => FALSE + falsefalse = EXAMPLE_INPUT.copy() + falsefalse['source.abuse_contact'] = "test@test.eu" + falsefalse['comment'] = "I am NOT True in AND clause" + falsefalse_result = falsefalse.copy() + self.input_message = falsefalse + self.run_bot() + self.assertMessageEqual(0, falsefalse_result) def test_precedence(self): """ Test precedence of operators """ @@ -49,21 +122,125 @@ def test_string_inverse_regex_match(self): def test_numeric_equal_match(self): """ Test == numeric match """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_equal_match.sieve') + + # if match drop + numeric_match_true=EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy']=100.0 + self.input_message=numeric_match_true + self.run_bot() + self.assertOutputQueueLen(0) + + # if doesn't match keep + numeric_match_false=EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy']=50.0 + self.input_message=numeric_match_false + self.run_bot() + self.assertMessageEqual(0,numeric_match_false) def test_numeric_not_equal_match(self): """ Test != numeric match """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_not_equal_match.sieve') + + # if not equal drop + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 50.0 + self.input_message = numeric_match_false + self.run_bot() + self.assertOutputQueueLen(0) + + # if equal keep + numeric_match_true = EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy'] = 100 + self.input_message = numeric_match_true + self.run_bot() + self.assertMessageEqual(0, numeric_match_true) def test_numeric_less_than_match(self): """ Test < numeric match """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_less_than_match.sieve') + + # if less than drop + numeric_match_true = EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy'] = 50.0 + self.input_message = numeric_match_true + self.run_bot() + self.assertOutputQueueLen(0) + + # if greater than keep + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 99.5 + self.input_message = numeric_match_false + self.run_bot() + self.assertMessageEqual(0, numeric_match_false) def test_numeric_less_than_or_equal_match(self): """ Test <= numeric match """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_less_than_or_equal_match.sieve') + + # if less than drop + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 40.0 + self.input_message = numeric_match_false + self.run_bot() + self.assertOutputQueueLen(0) + + # if equal drop + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 90 + self.input_message = numeric_match_false + self.run_bot() + self.assertOutputQueueLen(0) + + # if greater than keep + numeric_match_true = EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy'] = 95.0 + self.input_message = numeric_match_true + self.run_bot() + self.assertMessageEqual(0, numeric_match_true) def test_numeric_greater_than_match(self): """ Test > numeric match """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_greater_than_match.sieve') + + # if greater than drop + numeric_match_true = EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy'] = 50.0 + self.input_message = numeric_match_true + self.run_bot() + self.assertOutputQueueLen(0) + + # if less than keep + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 35.5 + self.input_message = numeric_match_false + self.run_bot() + self.assertMessageEqual(0, numeric_match_false) def test_numeric_greater_than_or_equal_match(self): """ Test >= numeric match """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_greater_than_or_equal_match.sieve') + + # if less than keep + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 40.0 + self.input_message = numeric_match_false + self.run_bot() + self.assertMessageEqual(0, numeric_match_false) + + # if equal drop + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 90 + self.input_message = numeric_match_false + self.run_bot() + self.assertOutputQueueLen(0) + + # if greater than drop + numeric_match_true = EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy'] = 95.0 + self.input_message = numeric_match_true + self.run_bot() + self.assertOutputQueueLen(0) def test_exists_match(self): """ Test :exists match """ @@ -119,6 +296,22 @@ def test_modify(self): def test_remove(self): """ Test removing keys """ + def test_multiple_actions(self): + """ Test applying multiple actions in one rule """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_multiple_actions.sieve') + + event = EXAMPLE_INPUT.copy() + event['classification.type'] = 'unknown' + self.input_message = event + self.run_bot() + + expected_result = event.copy() + expected_result['source.ip'] = '127.0.0.2' + expected_result['comment'] = 'added' + del expected_result['classification.type'] + + self.assertMessageEqual(0, expected_result) + diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_and_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_and_match.sieve new file mode 100644 index 000000000..95a88911a --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_and_match.sieve @@ -0,0 +1,3 @@ +if ( source.abuse_contact == "abuse@example.com" && comment == "I am TRUE in AND clause" ) { + add! source.ip = "10.9.8.7" +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve new file mode 100644 index 000000000..605e1b694 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve @@ -0,0 +1,5 @@ +if :exists source.ip { + add comment = 'added' + modify source.ip = '127.0.0.2' + remove classification.type +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve new file mode 100644 index 000000000..27109ea08 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve @@ -0,0 +1 @@ +if feed.accuracy == 100.0 { drop } \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_match.sieve new file mode 100644 index 000000000..6da8c5a0b --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_match.sieve @@ -0,0 +1 @@ +if feed.accuracy > 40.0 { drop } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve new file mode 100644 index 000000000..294f57655 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve @@ -0,0 +1 @@ +if feed.accuracy >= 90 { drop } \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_match.sieve new file mode 100644 index 000000000..2c0a82435 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_match.sieve @@ -0,0 +1 @@ +if feed.accuracy < 98.0 { drop } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve new file mode 100644 index 000000000..beeac6cb9 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve @@ -0,0 +1 @@ +if feed.accuracy <= 90 { drop } \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve new file mode 100644 index 000000000..0c61006a5 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve @@ -0,0 +1 @@ +if feed.accuracy != 100.0 { drop } \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve new file mode 100644 index 000000000..08d9c8470 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve @@ -0,0 +1,3 @@ +if ( source.abuse_contact == "abuse@example.com" || comment == "I am TRUE in OR clause" ) { + modify source.ip = "10.9.8.7" +} From d06b1972fc412cbd2c9c4075df614107248baab0 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Wed, 16 Aug 2017 16:22:32 +0200 Subject: [PATCH 08/40] Fix antoinet/intelmq#5: added error and debug logging messages --- intelmq/bots/experts/sieve/expert.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index e9ccfdee5..835929762 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -25,8 +25,8 @@ def init(self): filename = os.path.join(os.path.dirname(__file__), 'sieve.tx') self.metamodel = metamodel_from_file(filename) except TextXError as e: - self.logger.error('Could not process sieve grammar file. Error in (%d, %d)', e.line, e.col) - self.logger.error(str(e)) # TODO: output textx exception message properly + self.logger.error('Could not process sieve grammar file. Error in (%d, %d).', e.line, e.col) + self.logger.error(str(e)) self.stop() # validate parameters @@ -37,8 +37,8 @@ def init(self): try: self.sieve = self.metamodel.model_from_file(self.parameters.file) except TextXError as e: - self.logger.error('Could not parse sieve file \'%r\', error in (%d, %d)', self.parameters.file, e.line, e.col) - self.logger.error(str(e)) # TODO: output textx exception message properly + self.logger.error('Could not parse sieve file %r, error in (%d, %d).', self.parameters.file, e.line, e.col) + self.logger.error(str(e)) self.stop() def process(self): @@ -46,8 +46,9 @@ def process(self): keep = False for rule in self.sieve.rules: - keep = SieveExpertBot.process_rule(rule, event) + keep = self.process_rule(rule, event) if not keep: + self.logger.debug('Dropped event based on rule at %s: %s.', self.get_position(rule), event) break if keep: @@ -55,11 +56,11 @@ def process(self): self.acknowledge_message() - @staticmethod - def process_rule(rule, event): + def process_rule(self, rule, event): match = SieveExpertBot.match_expression(rule.expr, event) keep = True if match: + self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule), event) for action in rule.actions: keep = SieveExpertBot.process_action(action.action, event) if not keep: @@ -158,5 +159,9 @@ def process_action(action, event): del event[action.key] return True + def get_position(self, entity): + """ returns the position (line,col) of an entity in the sieve file. """ + parser = self.metamodel.parser + return parser.pos_to_linecol(entity._tx_position) BOT = SieveExpertBot From 873a9fb465a0ea5303b39f08208131580e170617 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Wed, 16 Aug 2017 22:38:39 +0200 Subject: [PATCH 09/40] Implemented antoinet/intelmq#6: added keep action: stops processing and forwards message. --- intelmq/bots/experts/sieve/expert.py | 38 ++++++++++++------- intelmq/bots/experts/sieve/sieve.tx | 9 ++++- .../tests/bots/experts/sieve/test_expert.py | 34 +++++++++++++---- .../test_sieve_files/test_drop_event.sieve | 6 ++- .../test_sieve_files/test_keep_event.sieve | 9 ++++- 5 files changed, 72 insertions(+), 24 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 835929762..9fce14a1c 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -12,11 +12,18 @@ import os import intelmq.lib.exceptions as exceptions import re +from enum import Enum from intelmq.lib.bot import Bot from textx.metamodel import metamodel_from_file from textx.exceptions import TextXError +class Procedure(Enum): + CONTINUE = 1 # continue processing subsequent rules (default) + KEEP = 2 # stop processing and keep event + DROP = 3 # stop processing and drop event + + class SieveExpertBot(Bot): def init(self): @@ -44,28 +51,31 @@ def init(self): def process(self): event = self.receive_message() - keep = False + procedure = Procedure.CONTINUE for rule in self.sieve.rules: - keep = self.process_rule(rule, event) - if not keep: + procedure = self.process_rule(rule, event) + if procedure == Procedure.KEEP: + self.logger.debug('Stop processing based on rule at %s: %s.', self.get_position(rule), event) + break + elif procedure == Procedure.DROP: self.logger.debug('Dropped event based on rule at %s: %s.', self.get_position(rule), event) break - if keep: + if procedure != Procedure.DROP: self.send_message(event) self.acknowledge_message() def process_rule(self, rule, event): match = SieveExpertBot.match_expression(rule.expr, event) - keep = True + procedure = Procedure.CONTINUE if match: self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule), event) for action in rule.actions: - keep = SieveExpertBot.process_action(action.action, event) - if not keep: + procedure = SieveExpertBot.process_action(action.action, event) + if procedure != Procedure.CONTINUE: break - return keep + return procedure @staticmethod def match_expression(expr, event): @@ -107,7 +117,7 @@ def process_string_match(key, op, value, event): return SieveExpertBot.process_string_operator(event[key], op, value.value) elif value.__class__.__name__ == 'StringValueList': for val in value.values: - if SieveExpertBot.process_string_operator(event[key], op, val): + if SieveExpertBot.process_string_operator(event[key], op, val.value): return True return False @@ -133,7 +143,7 @@ def process_numeric_match(key, op, value, event): return SieveExpertBot.process_numeric_operator(event[key], op, value.value) elif value.__class__.__name__ == 'NumericValueList': for val in value.values: - if SieveExpertBot.process_numeric_operator(event[key], op, val): + if SieveExpertBot.process_numeric_operator(event[key], op, val.value): return True return False @@ -145,8 +155,10 @@ def process_numeric_operator(lhs, op, rhs): def process_action(action, event): print(type(event)) if action == 'drop': - return False - elif action.__class__.__name__ == 'AddAction': + return Procedure.DROP + elif action == 'keep': + return Procedure.KEEP + elif action.__class__.__name__ == 'AddAction': if action.key not in event: event.add(action.key, action.value) elif action.__class__.__name__ == 'AddForceAction': @@ -157,7 +169,7 @@ def process_action(action, event): elif action.__class__.__name__ == 'RemoveAction': if action.key in event: del event[action.key] - return True + return Procedure.CONTINUE def get_position(self, entity): """ returns the position (line,col) of an entity in the sieve file. """ diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 36f525f89..07ea2ea26 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -54,8 +54,15 @@ SingleNumericValue: value=NUMBER ; NumericValueList: '[' values+=SingleNumericValue[','] ']' ; -Action: action=DropAction | (action=AddAction | action=AddForceAction | action=ModifyAction | action=RemoveAction); +Action: action=DropAction + | ( action=KeepAction + | action=AddAction + | action=AddForceAction + | action=ModifyAction + | action=RemoveAction + ); DropAction: 'drop'; +KeepAction: 'keep'; AddAction: 'add' key=Key '=' value=STRING; // add key/value without overwriting existing key AddForceAction: 'add!' key=Key '=' value=STRING; // add key/value, overwriting existing key ModifyAction: 'modify' key=Key '=' value=STRING; // modify key/value, do not create if not exists diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 042498457..3d290f69f 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -13,6 +13,7 @@ "time.observation": "2017-01-01T00:00:00+00:00", } + class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for SieveExpertBot. @@ -104,21 +105,27 @@ def test_and_match(self): def test_precedence(self): """ Test precedence of operators """ + # TODO def test_string_equal_match(self): """ Test == string match """ + # TODO def test_string_not_equal_match(self): """ Test != string match """ + # TODO def test_string_contains_match(self): """ Test :contains string match """ + # TODO def test_string_regex_match(self): """ Test =~ string match """ + # TODO def test_string_inverse_regex_match(self): """ Test !~ string match """ + # TODO def test_numeric_equal_match(self): """ Test == numeric match """ @@ -244,18 +251,22 @@ def test_numeric_greater_than_or_equal_match(self): def test_exists_match(self): """ Test :exists match """ + # TODO def test_not_exists_match(self): """ Test :notexists match """ + # TODO def test_string_match_value_list(self): """ Test string match with StringValueList """ + # TODO def test_numeric_match_value_list(self): """ Test numeric match with StringValueList """ + # TODO def test_drop_event(self): - """ Test if matched event is dropped. """ + """ Test if matched event is dropped and processing is stopped. """ self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_drop_event.sieve') event1 = EXAMPLE_INPUT.copy() @@ -264,37 +275,46 @@ def test_drop_event(self): self.assertMessageEqual(0, event1) event2 = EXAMPLE_INPUT.copy() - event2['comment'] = "deleteme" + event2['comment'] = 'drop' self.input_message = event2 self.run_bot() self.assertOutputQueueLen(0) def test_keep_event(self): - """ Test if matched event is kept. """ + """ Test if matched event is kept and processing is stopped. """ self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_keep_event.sieve') event1 = EXAMPLE_INPUT.copy() + event1['comment'] = 'continue' self.input_message = event1 self.run_bot() - self.assertMessageEqual(0, event1) + expected1 = EXAMPLE_INPUT.copy() + expected1['comment'] = 'changed' + self.assertMessageEqual(0, expected1) event2 = EXAMPLE_INPUT.copy() - event2['comment'] = "keepme" + event2['comment'] = 'keep' self.input_message = event2 self.run_bot() - self.assertMessageEqual(0, event2) + expected2 = EXAMPLE_INPUT.copy() + expected2['comment'] = 'keep' + self.assertMessageEqual(0, expected2) def test_add(self): """ Test adding key/value pairs """ + # TODO def test_add_force(self): """ Test adding key/value pairs, overwriting existing key """ + # TODO def test_modify(self): """ Test modifying key/value pairs """ + # TODO def test_remove(self): """ Test removing keys """ + # TODO def test_multiple_actions(self): """ Test applying multiple actions in one rule """ @@ -313,7 +333,5 @@ def test_multiple_actions(self): self.assertMessageEqual(0, expected_result) - - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve index 49921b68c..023372a07 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve @@ -1,3 +1,7 @@ -if comment == 'deleteme' { +if comment == 'drop' { drop +} + +if source.ip == '127.0.0.1' { + modify comment = 'changed' } \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve index 9ce5be89e..932f37017 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve @@ -1,3 +1,10 @@ -if comment == 'keepme' { +if comment == 'continue' { +} +if comment == 'keep' { + keep +} + +if comment == ['continue', 'keep'] { + modify comment = 'changed' } \ No newline at end of file From 8446988f19f35020ba9254ccc6d4d95894c7420c Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Thu, 17 Aug 2017 15:13:35 +0200 Subject: [PATCH 10/40] Add else and elsif clauses in rules, fixes antoinet/intelmq#4 --- intelmq/bots/experts/sieve/expert.py | 35 +++- intelmq/bots/experts/sieve/sieve.tx | 9 +- intelmq/bots/experts/sieve/test.sieve | 2 + .../tests/bots/experts/sieve/test_expert.py | 154 ++++++++++++++++-- .../test_sieve_files/test_if_clause.sieve | 9 + .../test_if_elif_clause.sieve | 7 + .../test_if_elif_else_clause.sieve | 7 + .../test_if_else_clause.sieve | 5 + 8 files changed, 202 insertions(+), 26 deletions(-) create mode 100644 intelmq/bots/experts/sieve/test.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 9fce14a1c..82f999930 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -50,7 +50,6 @@ def init(self): def process(self): event = self.receive_message() - procedure = Procedure.CONTINUE for rule in self.sieve.rules: procedure = self.process_rule(rule, event) @@ -61,21 +60,41 @@ def process(self): self.logger.debug('Dropped event based on rule at %s: %s.', self.get_position(rule), event) break + # forwarding decision if procedure != Procedure.DROP: self.send_message(event) self.acknowledge_message() def process_rule(self, rule, event): - match = SieveExpertBot.match_expression(rule.expr, event) - procedure = Procedure.CONTINUE - if match: - self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule), event) - for action in rule.actions: + # process mandatory 'if' clause + if SieveExpertBot.match_expression(rule.if_.expr, event): + self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule.if_), event) + for action in rule.if_.actions: + procedure = SieveExpertBot.process_action(action.action, event) + if procedure != Procedure.CONTINUE: + return procedure + return Procedure.CONTINUE + + # process optional 'elif' clauses + for clause in rule.elif_: + if SieveExpertBot.match_expression(clause.expr, event): + self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(clause), event) + for action in clause.actions: + procedure = SieveExpertBot.process_action(action.action, event) + if procedure != Procedure.CONTINUE: + return procedure + return Procedure.CONTINUE + + # process optional 'else' clause + if rule.else_: + self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule.else_), event) + for action in rule.else_.actions: procedure = SieveExpertBot.process_action(action.action, event) if procedure != Procedure.CONTINUE: - break - return procedure + return procedure + + return Procedure.CONTINUE @staticmethod def match_expression(expr, event): diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 07ea2ea26..3db8ac855 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -5,11 +5,10 @@ SieveModel: rules*=Rule ; -Rule: - 'if' expr=Expression '{' - actions*=Action - '}' -; +Rule: if_=IfClause (elif_=ElifClause)* (else_=ElseClause)?; +IfClause: 'if' expr=Expression '{' actions*=Action '}'; +ElifClause: 'elif' expr=Expression '{' actions*=Action '}'; +ElseClause: 'else' '{' actions*=Action '}'; Expression: conj=Conjunction ('||' conj=Conjunction)*; Conjunction: cond=Condition ('&&' cond=Condition)*; diff --git a/intelmq/bots/experts/sieve/test.sieve b/intelmq/bots/experts/sieve/test.sieve new file mode 100644 index 000000000..f246e78b3 --- /dev/null +++ b/intelmq/bots/experts/sieve/test.sieve @@ -0,0 +1,2 @@ +if source.ip == '127.0.0.1' {} + diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 3d290f69f..3f0647a09 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -23,6 +23,128 @@ class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): def set_bot(cls): cls.bot_reference = SieveExpertBot + def test_if_clause(self): + """ Test processing of subsequent if clauses. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_if_clause.sieve') + + # assert first if clause is matched + event1 = EXAMPLE_INPUT.copy() + event1['comment'] = 'changeme' + expected1 = EXAMPLE_INPUT.copy() + expected1['comment'] = 'changed' + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, expected1) + + # assert second if clause is matched + event2 = EXAMPLE_INPUT.copy() + event2['source.ip'] = '192.168.0.1' + expected2 = EXAMPLE_INPUT.copy() + expected2['source.ip'] = '192.168.0.2' + self.input_message = event2 + self.run_bot() + self.assertMessageEqual(0, expected2) + + # assert both if clauses are matched + event3 = EXAMPLE_INPUT.copy() + event3['comment'] = 'changeme' + event3['source.ip'] = '192.168.0.1' + expected3 = EXAMPLE_INPUT.copy() + expected3['comment'] = 'changed' + expected3['source.ip'] = '192.168.0.2' + self.input_message = event3 + self.run_bot() + self.assertMessageEqual(0, expected3) + + # assert none of the if clauses matched + event4 = EXAMPLE_INPUT.copy() + self.input_message = event4 + self.run_bot() + self.assertMessageEqual(0, event4) + + def test_if_else_clause(self): + """ Test processing else clause. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_if_else_clause.sieve') + + # assert that event matches if clause + event1 = EXAMPLE_INPUT.copy() + event1['comment'] = 'match' + expected1 = EXAMPLE_INPUT.copy() + expected1['comment'] = 'matched' + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, expected1) + + # assert that action in else clause is applied + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = 'foobar' + expected2 = EXAMPLE_INPUT.copy() + expected2['comment'] = 'notmatched' + self.input_message = event2 + self.run_bot() + self.assertMessageEqual(0, expected2) + + def test_if_elif_clause(self): + """ Test processing elif clauses. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_if_elif_clause.sieve') + + # test match if clause + event = EXAMPLE_INPUT.copy() + event['comment'] = 'match1' + expected = EXAMPLE_INPUT.copy() + expected['comment'] = 'changed1' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # test match first elif clause + event['comment'] = 'match2' + expected['comment'] = 'changed2' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # test match second elif clause + event['comment'] = 'match3' + expected['comment'] = 'changed3' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # test no match + event['comment'] = 'foobar' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + def test_if_elif_else_clause(self): + """ Test processing if, elif, and else clause. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_if_elif_else_clause.sieve') + + # test match if clause + event = EXAMPLE_INPUT.copy() + event['comment'] = 'match1' + expected = EXAMPLE_INPUT.copy() + expected['comment'] = 'changed1' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # test match elif clause + event['comment'] = 'match2' + expected['comment'] = 'changed2' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # test match else clause + event['comment'] = 'match3' + expected['comment'] = 'changed3' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + def test_or_match(self): """ Test Or Operator in match""" self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_or_match.sieve') @@ -129,25 +251,27 @@ def test_string_inverse_regex_match(self): def test_numeric_equal_match(self): """ Test == numeric match """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_equal_match.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_equal_match.sieve') # if match drop - numeric_match_true=EXAMPLE_INPUT.copy() - numeric_match_true['feed.accuracy']=100.0 - self.input_message=numeric_match_true + numeric_match_true = EXAMPLE_INPUT.copy() + numeric_match_true['feed.accuracy'] = 100.0 + self.input_message = numeric_match_true self.run_bot() self.assertOutputQueueLen(0) # if doesn't match keep - numeric_match_false=EXAMPLE_INPUT.copy() - numeric_match_false['feed.accuracy']=50.0 - self.input_message=numeric_match_false + numeric_match_false = EXAMPLE_INPUT.copy() + numeric_match_false['feed.accuracy'] = 50.0 + self.input_message = numeric_match_false self.run_bot() - self.assertMessageEqual(0,numeric_match_false) + self.assertMessageEqual(0, numeric_match_false) def test_numeric_not_equal_match(self): """ Test != numeric match """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_not_equal_match.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_not_equal_match.sieve') # if not equal drop numeric_match_false = EXAMPLE_INPUT.copy() @@ -165,7 +289,8 @@ def test_numeric_not_equal_match(self): def test_numeric_less_than_match(self): """ Test < numeric match """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_less_than_match.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_less_than_match.sieve') # if less than drop numeric_match_true = EXAMPLE_INPUT.copy() @@ -183,7 +308,8 @@ def test_numeric_less_than_match(self): def test_numeric_less_than_or_equal_match(self): """ Test <= numeric match """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_less_than_or_equal_match.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_less_than_or_equal_match.sieve') # if less than drop numeric_match_false = EXAMPLE_INPUT.copy() @@ -208,7 +334,8 @@ def test_numeric_less_than_or_equal_match(self): def test_numeric_greater_than_match(self): """ Test > numeric match """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_greater_than_match.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_greater_than_match.sieve') # if greater than drop numeric_match_true = EXAMPLE_INPUT.copy() @@ -226,7 +353,8 @@ def test_numeric_greater_than_match(self): def test_numeric_greater_than_or_equal_match(self): """ Test >= numeric match """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_greater_than_or_equal_match.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_greater_than_or_equal_match.sieve') # if less than keep numeric_match_false = EXAMPLE_INPUT.copy() diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve new file mode 100644 index 000000000..df307ee85 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve @@ -0,0 +1,9 @@ +if comment == 'changeme' { + modify comment = 'changed' +} + +if source.ip == '192.168.0.1' { + modify source.ip = '192.168.0.2' +} + + diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve new file mode 100644 index 000000000..61f9ac8ec --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve @@ -0,0 +1,7 @@ +if comment == 'match1' { + modify comment = 'changed1' +} elif comment == 'match2' { + modify comment = 'changed2' +} elif comment == 'match3' { + modify comment = 'changed3' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve new file mode 100644 index 000000000..fe8b0d36c --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve @@ -0,0 +1,7 @@ +if comment == 'match1' { + modify comment = 'changed1' +} elif comment == 'match2' { + modify comment = 'changed2' +} else { + modify comment = 'changed3' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve new file mode 100644 index 000000000..a3bfa21d5 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve @@ -0,0 +1,5 @@ +if comment == 'match' { + modify comment = 'matched' +} else { + modify comment = 'notmatched' +} \ No newline at end of file From fda9cad071152130e0845baba9cbec0f08a60d21 Mon Sep 17 00:00:00 2001 From: Kevin Holvoet Date: Thu, 17 Aug 2017 11:05:35 +0200 Subject: [PATCH 11/40] Add new test to test ADD action --- intelmq/bots/experts/sieve/expert.py | 10 +- .../tests/bots/experts/sieve/test_expert.py | 102 ++++++++++++++++-- .../sieve/test_sieve_files/test_add.sieve | 3 + .../test_sieve_files/test_add_force.sieve | 7 ++ .../test_sieve_files/test_exists_match.sieve | 3 + .../sieve/test_sieve_files/test_modify.sieve | 6 ++ .../test_notexists_match.sieve | 3 + 7 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 82f999930..7c34d7e0a 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -2,7 +2,6 @@ """ SieveExpertBot filters and modifies events based on a specification language similar to mail sieve. -TODO: Document possible necessary configurations. Parameters: file: string """ @@ -114,7 +113,7 @@ def process_conjunction(conj, event): def process_condition(cond, event): match = cond.match if match.__class__.__name__ == 'ExistMatch': - return SieveExpertBot.process_exist_match(match.key, event) + return SieveExpertBot.process_exist_match(match.key, match.op, event) elif match.__class__.__name__ == 'StringMatch': return SieveExpertBot.process_string_match(match.key, match.op, match.value, event) elif match.__class__.__name__ == 'NumericMatch': @@ -124,8 +123,11 @@ def process_condition(cond, event): pass @staticmethod - def process_exist_match(key, event): - return key in event + def process_exist_match(key, op, event): + if op == ':exists': + return key in event + elif op == ':notexists': + return key not in event @staticmethod def process_string_match(key, op, value, event): diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 3f0647a09..cbaf245af 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -6,7 +6,6 @@ import intelmq.lib.test as test from intelmq.bots.experts.sieve.expert import SieveExpertBot - EXAMPLE_INPUT = {"__type": "Event", "source.ip": "127.0.0.1", "source.abuse_contact": "abuse@example.com", @@ -379,11 +378,41 @@ def test_numeric_greater_than_or_equal_match(self): def test_exists_match(self): """ Test :exists match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_exists_match.sieve') + + # positive test + event = EXAMPLE_INPUT.copy() + event['source.fqdn'] = 'www.example.com' + expected = event.copy() + expected['comment'] = 'I think therefore I am.' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test + event = EXAMPLE_INPUT.copy() + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_not_exists_match(self): """ Test :notexists match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_notexists_match.sieve') + + # positive test + event = EXAMPLE_INPUT.copy() + expected = event.copy() + expected['comment'] = 'I think therefore I am.' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test + event = EXAMPLE_INPUT.copy() + event['source.fqdn'] = 'www.example.com' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_string_match_value_list(self): """ Test string match with StringValueList """ @@ -430,15 +459,75 @@ def test_keep_event(self): def test_add(self): """ Test adding key/value pairs """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_add.sieve') + + # If doesn't match, nothing should have changed + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + # If expression matches, destination.ip field is added + event1['comment'] = 'add field' + result = event1.copy() + result['destination.ip'] = '150.50.50.10' + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, result) def test_add_force(self): """ Test adding key/value pairs, overwriting existing key """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_add_force.sieve') + + # If doesn't match, nothing should have changed + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + # If expression matches, destination.ip field is added as new field + event1['comment'] = 'add force new field' + result = event1.copy() + result['destination.ip'] = '150.50.50.10' + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, result) + + # If expression matches, destination.ip field is added as new field + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = 'add force existing fields' + result2 = event2.copy() + result2['destination.ip'] = '200.10.9.7' + result2['source.ip'] = "10.9.8.7" + self.input_message = event2 + self.run_bot() + self.assertMessageEqual(0, result2) def test_modify(self): """ Test modifying key/value pairs """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_modify.sieve') + + # If doesn't match, nothing should have changed + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + # If expression matches && parameter doesn't exists, nothing changes + event1['comment'] = 'modify new parameter' + result = event1.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, result) + + # If expression matches && parameter exists, source.ip changed + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = 'modify existing parameter' + result2 = event2.copy() + result2['source.ip'] = '10.9.8.7' + self.input_message = event2 + self.run_bot() + self.assertMessageEqual(0, result2) def test_remove(self): """ Test removing keys """ @@ -460,6 +549,5 @@ def test_multiple_actions(self): self.assertMessageEqual(0, expected_result) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve new file mode 100644 index 000000000..bc7dbdcc3 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve @@ -0,0 +1,3 @@ +if comment == 'add field' { + add destination.ip="150.50.50.10" +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve new file mode 100644 index 000000000..45939f238 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve @@ -0,0 +1,7 @@ +if comment == 'add force new field' { + add! destination.ip = "150.50.50.10" +} +if comment == 'add force existing fields' { + add! destination.ip = "200.10.9.7" + add! source.ip = "10.9.8.7" +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve new file mode 100644 index 000000000..fc070c292 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve @@ -0,0 +1,3 @@ +if :exists source.fqdn { + add comment = "I think therefore I am." +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve new file mode 100644 index 000000000..11b1cfa02 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve @@ -0,0 +1,6 @@ +if comment == 'modify new parameter' { + modify destination.ip="100.99.88.77" +} +if comment == 'modify existing parameter' { + modify source.ip="10.9.8.7" +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve new file mode 100644 index 000000000..794dadc62 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve @@ -0,0 +1,3 @@ +if :notexists source.fqdn { + add comment = "I think therefore I am." +} \ No newline at end of file From 99b6cc26cff7f0edb27a2267e3d53c55bbba515b Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 18 Aug 2017 09:07:54 +0200 Subject: [PATCH 12/40] Document usage/features in README.md, antoinet/intelmq#9 --- intelmq/bots/experts/sieve/README.md | 110 ++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 10 deletions(-) diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index 7434d0152..079113e44 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -10,30 +10,120 @@ rule's conditions, the corresponding actions are performed. Actions can specify whether the event should be kept or dropped in the pipeline (filtering actions) or if keys and values should be changed (modification actions). + ## Examples -The following excerpts illustrate the basic features of the sieve file format. -### Filtering based on event properties +The following excerpts illustrate some of the basic features of the sieve file +format: ``` -if source.ip == '127.0.0.1' { - drop +if :exists source.fqdn { + keep // aborts processing of subsequent rules and forwards the event. } + if :notexists source.abuse_contact || source.abuse_contact =~ '.*@example.com' { - drop + drop // aborts processing of subsequent rules and drops the event. } -``` -### Modification based on event properties -``` if classification.type == ['phishing', 'malware'] && source.fqdn =~ '.*\.(ch|li)$' { add comment = 'domainabuse' - modify classification.taxonomy = 'fraud' - remove extra.comments + keep +} elsif classification.type == 'scanner' { + add comment = 'ignore' + drop +} else { + remove comment } ``` + ## Parameters + +The sieve bot only takes one parameter: * `file` - filesystem path the the sieve file + + +## Reference + +### Sieve File Structure + +The sieve file contains an arbitrary number of rules of the form: + +``` +if EXPRESSION { + ACTIONS +} elif EXPRESSION { + ACTIONS +} else { + ACTIONS +} +``` + + +### Expressions + +Each rule specifies on or more expressions to match an event based on its keys +and values. Event keys are specified as strings without quotes. String values +must be enclosed in single quotes. Numeric values can be specified as integers +or floats and are unquoted. Following operators may be used to match events: + + * `:exists` and `:notexists` match if a given key exists, for example: + + ```if :exists source.fqdn { ... }``` + + * `==` and `!=` match for equality of strings and numbers, for example: + + ```if feed.name != 'acme-security' || feed.accuracy == 100 { ... }``` + + * `:contains` matches on substrings. + + * `=~` matches strings based on the given regex. `!~` is the inverse regex + match. + + * Numerical comparisons are evaluated with `<`, `<=`, `>`, `>=`. + + * Values to match against can also be specified as list, in which case any one + of the values will result in a match: + + ```if source.ip == ['8.8.8.8', '8.8.4.4'] { ... }``` + + In this case, the event will match if it contains a key `source.ip` with + either value `8.8.8.8` or `8.8.4.4`. + + +### Actions + +If part of a rule matches the given conditions, the actions enclosed in `{` and +`}` are applied. By default, all events that are matched or not matched by rules +in the sieve file will be forwarded to the next bot in the pipeline, unless the +`drop` action is applied. + + * `add` adds a key value pair to the event. This action only applies if the key + is not yet defined in the event. If the key is already defined, the action is + ignored. Example: + + ```add comment = 'hello, world'``` + + * `add!` same as above, but will force overwrite the key in the event. + + * `modify` modifies an existing value for a key. Only applies if the key is +already defined. If the key is not defined in the event, this action is ignored. +Example: + + ```modify feed.accuary = 50``` + + * `remove` removes a key/value from the event. Action is ignored if the key is + not defined in the event. Example: + + ```remove extra.comments``` + + * `keep` marks the event to be forwarded to the next bot in the pipeline + (same as the default behaviour), but in addition the sieve file processing is + interrupted upon reaching this action. + + * `drop` marks the event to be dropped. The event will not be forwarded to the + next bot in the pipeline. The sieve file processing is interrupted upon + reaching this action. No other actions may be specified besides the `drop` + action within `{` and `}`. From 6b2759fe0c1eca370b8781dbe637bca6afd5ab73 Mon Sep 17 00:00:00 2001 From: Helder Date: Thu, 17 Aug 2017 15:19:08 +0100 Subject: [PATCH 13/40] Implementation of Numeric match value list and String match value list --- .../tests/bots/experts/sieve/test_expert.py | 81 +++++++++++++------ .../test_numeric_match_value_list.sieve | 11 +++ .../test_string_match_value_list.sieve | 11 +++ 3 files changed, 77 insertions(+), 26 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index cbaf245af..20669cee4 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -416,11 +416,62 @@ def test_not_exists_match(self): def test_string_match_value_list(self): """ Test string match with StringValueList """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_string_match_value_list.sieve') + + # Match the first rule + string_value_list_match_1 = EXAMPLE_INPUT.copy() + string_value_list_match_1['classification.type'] = 'malware' + string_value_list_expected_result_1=string_value_list_match_1.copy() + string_value_list_expected_result_1['comment'] ='infected hosts' + self.input_message = string_value_list_match_1 + self.run_bot() + self.assertMessageEqual(0, string_value_list_expected_result_1) + + # Match the second rule + string_value_list_match_2 = EXAMPLE_INPUT.copy() + string_value_list_match_2['classification.type'] = 'c&c' + string_value_list_expected_result_2=string_value_list_match_2.copy() + string_value_list_expected_result_2['comment'] ='malicious server / service' + self.input_message = string_value_list_match_2 + self.run_bot() + self.assertMessageEqual(0, string_value_list_expected_result_2) + + #don't Match any rule + string_value_list_match_3 = EXAMPLE_INPUT.copy() + string_value_list_match_3['classification.type'] = 'blacklist' + self.input_message = string_value_list_match_3 + self.run_bot() + self.assertMessageEqual(0, string_value_list_match_3) + def test_numeric_match_value_list(self): - """ Test numeric match with StringValueList """ - # TODO + """ Test numeric match with NumericValueList """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_match_value_list.sieve') + + # Match the first rule + numeric_value_list_match_1 = EXAMPLE_INPUT.copy() + numeric_value_list_match_1['destination.asn'] = 6939 + numeric_value_list_expected_result_1 = numeric_value_list_match_1.copy() + numeric_value_list_expected_result_1['comment'] = 'Belongs to peering group' + self.input_message = numeric_value_list_match_1 + self.run_bot() + self.assertMessageEqual(0, numeric_value_list_expected_result_1) + + # Match the second rule + numeric_value_list_match_2 = EXAMPLE_INPUT.copy() + numeric_value_list_match_2['destination.asn'] = 1930 + numeric_value_list_expected_result_2 = numeric_value_list_match_2.copy() + numeric_value_list_expected_result_2['comment'] = 'Belongs constituency group' + self.input_message = numeric_value_list_match_2 + self.run_bot() + self.assertMessageEqual(0, numeric_value_list_expected_result_2) + + # don't Match any rule + numeric_value_list_match_3 = EXAMPLE_INPUT.copy() + numeric_value_list_match_3['destination.asn'] = 3356 + self.input_message = numeric_value_list_match_3 + self.run_bot() + self.assertMessageEqual(0, numeric_value_list_match_3) def test_drop_event(self): """ Test if matched event is dropped and processing is stopped. """ @@ -505,29 +556,7 @@ def test_add_force(self): def test_modify(self): """ Test modifying key/value pairs """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_modify.sieve') - - # If doesn't match, nothing should have changed - event1 = EXAMPLE_INPUT.copy() - self.input_message = event1 - self.run_bot() - self.assertMessageEqual(0, event1) - - # If expression matches && parameter doesn't exists, nothing changes - event1['comment'] = 'modify new parameter' - result = event1.copy() - self.input_message = event1 - self.run_bot() - self.assertMessageEqual(0, result) - - # If expression matches && parameter exists, source.ip changed - event2 = EXAMPLE_INPUT.copy() - event2['comment'] = 'modify existing parameter' - result2 = event2.copy() - result2['source.ip'] = '10.9.8.7' - self.input_message = event2 - self.run_bot() - self.assertMessageEqual(0, result2) + # TODO def test_remove(self): """ Test removing keys """ diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve new file mode 100644 index 000000000..765ca934e --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve @@ -0,0 +1,11 @@ +if destination.asn == [20965,6939,8220] { + + add comment='Belongs to peering group' + +} + +if destination.asn == [1930,199155] { + + add comment='Belongs constituency group' + +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve new file mode 100644 index 000000000..c75877367 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve @@ -0,0 +1,11 @@ +if classification.type == ['botnet drone','malware','ransomware'] { + +add comment='infected hosts' + +} + +if classification.type == ['c&c','malware configuration'] { + +add comment='malicious server / service' + +} \ No newline at end of file From 706819f7ad006b7c9ee105669698a2cdb5b8d893 Mon Sep 17 00:00:00 2001 From: Kevin Holvoet Date: Thu, 17 Aug 2017 16:11:22 +0200 Subject: [PATCH 14/40] Add tests for REMOVE action --- .../tests/bots/experts/sieve/test_expert.py | 48 ++++++++++++++++++- .../sieve/test_sieve_files/test_remove.sieve | 3 ++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 20669cee4..e4f3f2440 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -556,11 +556,55 @@ def test_add_force(self): def test_modify(self): """ Test modifying key/value pairs """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_modify.sieve') + + # If doesn't match, nothing should have changed + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + # If expression matches && parameter doesn't exists, nothing changes + event1['comment'] = 'modify new parameter' + result = event1.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, result) + + # If expression matches && parameter exists, source.ip changed + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = 'modify existing parameter' + result2 = event2.copy() + result2['source.ip'] = '10.9.8.7' + self.input_message = event2 + self.run_bot() + self.assertMessageEqual(0, result2) def test_remove(self): """ Test removing keys """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_remove.sieve') + + # If doesn't match, nothing should have changed + event1 = EXAMPLE_INPUT.copy() + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, event1) + + # If expression matches && parameter exists, parameter is removed + event1['comment'] = 'remove parameter' + result = event1.copy() + event1['destination.ip'] = '192.168.10.1' + self.input_message = event1 + self.run_bot() + self.assertMessageEqual(0, result) + + # If expression matches && parameter doesn't exist, nothing happens + event2 = EXAMPLE_INPUT.copy() + event2['comment'] = 'remove parameter' + result2 = event2.copy() + self.input_message = event2 + self.run_bot() + self.assertMessageEqual(0, result2) def test_multiple_actions(self): """ Test applying multiple actions in one rule """ diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve new file mode 100644 index 000000000..f59658956 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve @@ -0,0 +1,3 @@ +if comment == 'remove parameter' { + remove destination.ip +} \ No newline at end of file From f5cc29cfff239e414d300a1e745d380d3f359680 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 18 Aug 2017 09:24:11 +0200 Subject: [PATCH 15/40] Added section about comments, antoinet/intelmq#9 --- intelmq/bots/experts/sieve/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index 079113e44..d76a45748 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -127,3 +127,8 @@ Example: next bot in the pipeline. The sieve file processing is interrupted upon reaching this action. No other actions may be specified besides the `drop` action within `{` and `}`. + + + ### Comments + + Comments may be used in the sieve file: all characters after `//` and until the end of the line will be ignored. From 3f48d8fafda5631346363e57fb5f71b6921d7d90 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 18 Aug 2017 10:37:41 +0200 Subject: [PATCH 16/40] Added command to validate sieve files, antoinet/intelmq#10 --- intelmq/bots/experts/sieve/README.md | 21 +++++++++- intelmq/bots/experts/sieve/filter.sieve | 53 ------------------------ intelmq/bots/experts/sieve/test.sieve | 2 - intelmq/bots/experts/sieve/validator.py | 54 +++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 57 deletions(-) delete mode 100644 intelmq/bots/experts/sieve/filter.sieve delete mode 100644 intelmq/bots/experts/sieve/test.sieve create mode 100644 intelmq/bots/experts/sieve/validator.py diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index d76a45748..76d392bb6 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -129,6 +129,23 @@ Example: action within `{` and `}`. - ### Comments +### Comments - Comments may be used in the sieve file: all characters after `//` and until the end of the line will be ignored. +Comments may be used in the sieve file: all characters after `//` and until the end of the line will be ignored. + + +## Validating a sieve file + +Use the following command to validate your sieve files: +``` +$ python intelmq/bots/experts/sieve/validator.py -h +usage: validator.py [-h] sievefile + +Validates the syntax of sievebot files. + +positional arguments: + sievefile Sieve file + +optional arguments: + -h, --help show this help message and exit +``` diff --git a/intelmq/bots/experts/sieve/filter.sieve b/intelmq/bots/experts/sieve/filter.sieve deleted file mode 100644 index 88a4f7d5c..000000000 --- a/intelmq/bots/experts/sieve/filter.sieve +++ /dev/null @@ -1,53 +0,0 @@ - -if allof ('source.ip' = '192.168.13.13', 'feed.name' :contains 'cymru') { reject;keep } - -if source.ip == ['123.123.123.123', '234.234.234.234'] { reject } - -if source.ip == '123.123.123.123' || source.ip == '234.234.234.234' - -// string equality operator == -// string inequality operator != -// regex match =~ -// regex not match !~ - -// or operator: || -// and operator: && - -if source.ip ~= '[' - -if anumber == 3 { keep; drop; keep } // watch out for this - -if :exists source.asn { .... } -if :!exists source.asn { ... } - -if source.ip == '123.123.123.123' { - add source.asn = 'AS559'; // add key/value only if key not present yet - add! source.asn = 'AS559'; // force overwrite if exists - modify source.asn = 'AS559'; // edit key/value only if key is present - remove source.asn; -} - - - -event = { - source.ip = '123.123.123.123', - source.asn = 'AS559' -} - -event2 = { - source.ip = '234.234.234.234' -} - - -if source.asn == 'AS559' { // do not match if key doesn't exist - -} - -// simulate ':exists' operator -if source.asn =~ /.*/ { ... } // should only match if key 'source.asn' exists - -// simulate ':notexists' operator? -if source.asn !~ /.*/ { ... } // will not match because key does not exist - - - diff --git a/intelmq/bots/experts/sieve/test.sieve b/intelmq/bots/experts/sieve/test.sieve deleted file mode 100644 index f246e78b3..000000000 --- a/intelmq/bots/experts/sieve/test.sieve +++ /dev/null @@ -1,2 +0,0 @@ -if source.ip == '127.0.0.1' {} - diff --git a/intelmq/bots/experts/sieve/validator.py b/intelmq/bots/experts/sieve/validator.py new file mode 100644 index 000000000..0f840edc6 --- /dev/null +++ b/intelmq/bots/experts/sieve/validator.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import argparse +import logging +from textx.metamodel import metamodel_from_file +from textx.exceptions import TextXError + + +class Validator(object): + + def __init__(self): + self.logger = logging.getLogger() + + grammarfile = os.path.join(os.path.dirname(__file__), 'sieve.tx') + if not os.path.exists(grammarfile): + self.logger.error('Sieve grammar file not found: %r.', grammarfile) + return + + try: + self.metamodel = metamodel_from_file(grammarfile) + except TextXError as e: + self.logger.error('Could not process sieve grammar file. Error in (%d, %d).', e.line, e.col) + self.logger.error(str(e)) + + def parse(self, filename): + if not self.metamodel: + self.logger.error('Metamodel not loaded.') + return + + if not os.path.exists(filename): + self.logger.error('File does not exist: %r', filename) + return + + try: + self.metamodel.model_from_file(filename) + except TextXError as e: + self.logger.error('Could not process sieve file %r.', filename) + self.logger.error('Error in (%d, %d).', e.line, e.col) + self.logger.error(str(e)) + return + + self.logger.info('Sieve file %r parsed successfully.', filename) + +if __name__ == "__main__": + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + parser = argparse.ArgumentParser(description="Validates the syntax of sievebot files.") + parser.add_argument('sievefile', help='Sieve file') + + args = parser.parse_args() + + validator = Validator() + validator.parse(args.sievefile) \ No newline at end of file From 40dda0d1953d2c4cd2c5ea0387101f8a01d27d9a Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Mon, 21 Aug 2017 10:29:34 +0200 Subject: [PATCH 17/40] Add 'ip in netblock' match, antoinet/intelmq#11 --- intelmq/bots/experts/sieve/README.md | 14 ++- intelmq/bots/experts/sieve/expert.py | 92 ++++++++++++------- intelmq/bots/experts/sieve/sieve.tx | 8 +- .../tests/bots/experts/sieve/test_expert.py | 68 ++++++++++++++ .../test_ip_range_list_match.sieve | 3 + .../test_ip_range_match.sieve | 18 ++++ 6 files changed, 164 insertions(+), 39 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index 76d392bb6..9a0c8167e 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -26,12 +26,15 @@ if :notexists source.abuse_contact || source.abuse_contact =~ '.*@example.com' { drop // aborts processing of subsequent rules and drops the event. } +if source.ip << 192.0.0.0/24 { + add! comment = 'bogon' +} if classification.type == ['phishing', 'malware'] && source.fqdn =~ '.*\.(ch|li)$' { - add comment = 'domainabuse' + add! comment = 'domainabuse' keep } elsif classification.type == 'scanner' { - add comment = 'ignore' + add! comment = 'ignore' drop } else { remove comment @@ -67,7 +70,8 @@ if EXPRESSION { Each rule specifies on or more expressions to match an event based on its keys and values. Event keys are specified as strings without quotes. String values must be enclosed in single quotes. Numeric values can be specified as integers -or floats and are unquoted. Following operators may be used to match events: +or floats and are unquoted. IP addresses and network ranges (IPv4 and IPv6) are +specified with quotes. Following operators may be used to match events: * `:exists` and `:notexists` match if a given key exists, for example: @@ -84,6 +88,10 @@ or floats and are unquoted. Following operators may be used to match events: * Numerical comparisons are evaluated with `<`, `<=`, `>`, `>=`. + * `<<` matches if an IP address is contained in the specified network range: + + ```if source.ip << '10.0.0.0/8' { ... }``` + * Values to match against can also be specified as list, in which case any one of the values will result in a match: diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 7c34d7e0a..09be1e8e3 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -11,10 +11,11 @@ import os import intelmq.lib.exceptions as exceptions import re +import ipaddress from enum import Enum from intelmq.lib.bot import Bot from textx.metamodel import metamodel_from_file -from textx.exceptions import TextXError +from textx.exceptions import TextXError, TextXSemanticError class Procedure(Enum): @@ -30,6 +31,7 @@ def init(self): try: filename = os.path.join(os.path.dirname(__file__), 'sieve.tx') self.metamodel = metamodel_from_file(filename) + self.metamodel.register_obj_processors({'SingleIpRange': SieveExpertBot.validate_ip_range}) except TextXError as e: self.logger.error('Could not process sieve grammar file. Error in (%d, %d).', e.line, e.col) self.logger.error(str(e)) @@ -67,20 +69,20 @@ def process(self): def process_rule(self, rule, event): # process mandatory 'if' clause - if SieveExpertBot.match_expression(rule.if_.expr, event): + if self.match_expression(rule.if_.expr, event): self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule.if_), event) for action in rule.if_.actions: - procedure = SieveExpertBot.process_action(action.action, event) + procedure = self.process_action(action.action, event) if procedure != Procedure.CONTINUE: return procedure return Procedure.CONTINUE # process optional 'elif' clauses for clause in rule.elif_: - if SieveExpertBot.match_expression(clause.expr, event): + if self.match_expression(clause.expr, event): self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(clause), event) for action in clause.actions: - procedure = SieveExpertBot.process_action(action.action, event) + procedure = self.process_action(action.action, event) if procedure != Procedure.CONTINUE: return procedure return Procedure.CONTINUE @@ -89,61 +91,57 @@ def process_rule(self, rule, event): if rule.else_: self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule.else_), event) for action in rule.else_.actions: - procedure = SieveExpertBot.process_action(action.action, event) + procedure = self.process_action(action.action, event) if procedure != Procedure.CONTINUE: return procedure return Procedure.CONTINUE - @staticmethod - def match_expression(expr, event): + def match_expression(self, expr, event): for conj in expr.conj: - if SieveExpertBot.process_conjunction(conj, event): + if self.process_conjunction(conj, event): return True return False - @staticmethod - def process_conjunction(conj, event): + def process_conjunction(self, conj, event): for cond in conj.cond: - if not SieveExpertBot.process_condition(cond, event): + if not self.process_condition(cond, event): return False return True - @staticmethod - def process_condition(cond, event): + def process_condition(self, cond, event): match = cond.match if match.__class__.__name__ == 'ExistMatch': - return SieveExpertBot.process_exist_match(match.key, match.op, event) + return self.process_exist_match(match.key, match.op, event) elif match.__class__.__name__ == 'StringMatch': - return SieveExpertBot.process_string_match(match.key, match.op, match.value, event) + return self.process_string_match(match.key, match.op, match.value, event) elif match.__class__.__name__ == 'NumericMatch': - return SieveExpertBot.process_numeric_match(match.key, match.op, match.value, event) + return self.process_numeric_match(match.key, match.op, match.value, event) + elif match.__class__.__name__ == 'IpRangeMatch': + return self.process_ip_range_match(match.key, match.range, event) elif match.__class__.__name__ == 'Expression': - return SieveExpertBot.match_expression(match, event) + return self.match_expression(match, event) pass - @staticmethod - def process_exist_match(key, op, event): + def process_exist_match(self, key, op, event): if op == ':exists': return key in event elif op == ':notexists': return key not in event - @staticmethod - def process_string_match(key, op, value, event): + def process_string_match(self, key, op, value, event): if key not in event: return False if value.__class__.__name__ == 'SingleStringValue': - return SieveExpertBot.process_string_operator(event[key], op, value.value) + return self.process_string_operator(event[key], op, value.value) elif value.__class__.__name__ == 'StringValueList': for val in value.values: - if SieveExpertBot.process_string_operator(event[key], op, val.value): + if self.process_string_operator(event[key], op, val.value): return True return False - @staticmethod - def process_string_operator(lhs, op, rhs): + def process_string_operator(self, lhs, op, rhs): if op == '==': return lhs == rhs elif op == '!=': @@ -155,25 +153,42 @@ def process_string_operator(lhs, op, rhs): elif op == '!~': return re.fullmatch(rhs, lhs) is None - @staticmethod - def process_numeric_match(key, op, value, event): + def process_numeric_match(self, key, op, value, event): if key not in event: return False if value.__class__.__name__ == 'SingleNumericValue': - return SieveExpertBot.process_numeric_operator(event[key], op, value.value) + return self.process_numeric_operator(event[key], op, value.value) elif value.__class__.__name__ == 'NumericValueList': for val in value.values: - if SieveExpertBot.process_numeric_operator(event[key], op, val.value): + if self.process_numeric_operator(event[key], op, val.value): return True return False - @staticmethod - def process_numeric_operator(lhs, op, rhs): - return eval(str(lhs) + op + str(rhs)) + def process_numeric_operator(self, lhs, op, rhs): + return eval(str(lhs) + op + str(rhs)) # TODO graceful error handling - @staticmethod - def process_action(action, event): + def process_ip_range_match(self, key, ip_range, event): + if key not in event: + return False + + try: + addr = ipaddress.ip_address(event[key]) + except ValueError: + self.logger.warning("Could not parse IP address %s=%s in %s.", key, event[key], event) + return False + + if ip_range.__class__.__name__ == 'SingleIpRange': + network = ipaddress.ip_network(ip_range.value) + return addr in network + elif ip_range.__class__.__name__ == 'IpRangeList': + for val in ip_range.values: + network = ipaddress.ip_network(val.value) + if addr in network: + return True + return False + + def process_action(self, action, event): print(type(event)) if action == 'drop': return Procedure.DROP @@ -197,4 +212,11 @@ def get_position(self, entity): parser = self.metamodel.parser return parser.pos_to_linecol(entity._tx_position) + @staticmethod + def validate_ip_range(ip_range): + try: + ipaddress.ip_network(ip_range.value) + except ValueError: + raise TextXSemanticError('Invalid ip range: %s.', ip_range.value) + BOT = SieveExpertBot diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 3db8ac855..7fd8ce444 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -15,6 +15,7 @@ Conjunction: cond=Condition ('&&' cond=Condition)*; Condition: match=StringMatch | match=NumericMatch + | match=IpRangeMatch | match=ExistMatch | ( '(' match=Expression ')' ) ; @@ -39,10 +40,15 @@ NumericOperator: | '>' // greater than ; +IpRangeMatch: key=Key '<<' range=IpRange; +IpRange: SingleIpRange | IpRangeList; +SingleIpRange: value=STRING; +IpRangeList: '[' values+=SingleIpRange[','] ']' ; + ExistMatch: op=ExistOperator key=Key; ExistOperator: ':exists' | ':notexists'; -Key: /[a-z_.]+/; +Key: /[a-z_\.]+/; StringValue: SingleStringValue | StringValueList ; SingleStringValue: value=STRING ; diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index e4f3f2440..9ef060619 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -622,5 +622,73 @@ def test_multiple_actions(self): self.assertMessageEqual(0, expected_result) + def test_ip_range_match(self): + """ Test IP range match operator. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_ip_range_match.sieve') + + # match /24 network + event = EXAMPLE_INPUT.copy() + event['source.ip'] = '192.0.0.1' + expected = event.copy() + expected['comment'] = 'bogon1' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # match /16 network + event = EXAMPLE_INPUT.copy() + event['source.ip'] = '192.0.200.1' + expected = event.copy() + expected['comment'] = 'bogon2' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # no match + event = EXAMPLE_INPUT.copy() + event['source.ip'] = '192.168.0.1' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + # IPv6 address + event = EXAMPLE_INPUT.copy() + event['source.ip'] = '2001:620:0:ff::56' + expected = event.copy() + expected['comment'] = 'SWITCH' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # invalid address in event should not match + event = EXAMPLE_INPUT.copy() + event['comment'] = '300.300.300.300' + self.input_message = event + self.run_bot() + self.assertLogMatches(pattern='^Could not parse IP address', levelname='WARNING') + self.assertMessageEqual(0, event) + + def test_ip_range_list_match(self): + """ Test IP range list match operator. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_ip_range_list_match.sieve') + + # positive test + event = EXAMPLE_INPUT.copy() + event['source.ip'] = '192.0.0.1' + expected = event.copy() + expected['comment'] = 'bogon' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test + event = EXAMPLE_INPUT.copy() + event['source.ip'] = '8.8.8.8' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve new file mode 100644 index 000000000..418582309 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve @@ -0,0 +1,3 @@ +if source.ip << ['192.0.0.0/24', '169.254.0.0/16'] { + add comment = 'bogon' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve new file mode 100644 index 000000000..2de1c457d --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve @@ -0,0 +1,18 @@ +if source.ip << '192.0.0.0/24' { + add! comment = 'bogon1' + keep +} + +if source.ip << '192.0.0.0/16' { + add! comment = 'bogon2' + keep +} + +if source.ip << '2001:620:0:0:0:0:0:0/29' { + add comment = 'SWITCH' + keep +} + +if comment << '10.0.0.0/8' { + add comment = 'valid' +} \ No newline at end of file From 76bc0dd244886fd0acfb8abf4f52a0e4e0dc36c4 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 18 Aug 2017 14:09:51 +0200 Subject: [PATCH 18/40] Update README.md --- intelmq/bots/experts/sieve/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index 9a0c8167e..a89f9ab7d 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -1,6 +1,6 @@ # Sieve Bot -The sieve bot is used to filter and/or modify bots based on a set of rules. The +The sieve bot is used to filter and/or modify events based on a set of rules. The rules are specified in an external configuration file and with a syntax similar to the [Sieve language](http://sieve.info/) used for mail filtering. @@ -90,7 +90,7 @@ specified with quotes. Following operators may be used to match events: * `<<` matches if an IP address is contained in the specified network range: - ```if source.ip << '10.0.0.0/8' { ... }``` + ```if source.ip << '10.0.0.0/8' { ... }``` * Values to match against can also be specified as list, in which case any one of the values will result in a match: From b73c48f6b3f7eb465a0b431adba8b922b2064150 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Tue, 22 Aug 2017 10:39:09 +0200 Subject: [PATCH 19/40] Added test missing test cases antoinet/intelmq#8 --- intelmq/bots/experts/sieve/expert.py | 2 +- .../tests/bots/experts/sieve/test_expert.py | 167 ++++++++++++++++-- .../test_sieve_files/test_precedence.sieve | 7 + .../test_string_contains_match.sieve | 3 + .../test_string_equal_match.sieve | 3 + .../test_string_inverse_regex_match.sieve | 3 + .../test_string_not_equal_match.sieve | 3 + .../test_string_regex_match.sieve | 3 + 8 files changed, 176 insertions(+), 15 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 09be1e8e3..3704e9bb4 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -131,7 +131,7 @@ def process_exist_match(self, key, op, event): def process_string_match(self, key, op, value, event): if key not in event: - return False + return op == '!=' or op == '!~' if value.__class__.__name__ == 'SingleStringValue': return self.process_string_operator(event[key], op, value.value) diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 9ef060619..dc90c5498 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -226,27 +226,165 @@ def test_and_match(self): def test_precedence(self): """ Test precedence of operators """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_precedence.sieve') + + # test && has higher precedence than || + event = EXAMPLE_INPUT.copy() + event['feed.provider'] = 'acme' + expected = event.copy() + expected['comment'] = 'match1' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # test round braces to change precedence + event = EXAMPLE_INPUT.copy() + event['source.abuse_contact'] = 'abuse@example.com' + event['source.ip'] = '5.6.7.8' + expected = event.copy() + expected['comment'] = 'match2' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) def test_string_equal_match(self): """ Test == string match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_equal_match.sieve') + + # positive test + event = EXAMPLE_INPUT.copy() + event['source.fqdn'] = 'www.example.com' + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test (key doesn't match) + event = EXAMPLE_INPUT.copy() + event['source.fqdn'] = 'www.hotmail.com' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + # negative test (key not defined) + event = EXAMPLE_INPUT.copy() + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_string_not_equal_match(self): """ Test != string match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_not_equal_match.sieve') + + # positive test (key mismatch) + event = EXAMPLE_INPUT.copy() + event['source.fqdn'] = 'mail.ru' + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # positive test (key undefined) + event = EXAMPLE_INPUT.copy() + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test + event = EXAMPLE_INPUT.copy() + event['source.fqdn'] = 'www.example.com' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_string_contains_match(self): """ Test :contains string match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_contains_match.sieve') + + # positive test + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'https://www.switch.ch/security/' + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test (key mismatch) + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'https://www.ripe.net/' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + # negative test (key undefined) + event = EXAMPLE_INPUT.copy() + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_string_regex_match(self): """ Test =~ string match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_regex_match.sieve') + + # positive test + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'https://www.switch.ch/security' + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test (key mismatch) + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'http://www.example.com' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + # negative test (key undefined) + event = EXAMPLE_INPUT.copy() + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_string_inverse_regex_match(self): """ Test !~ string match """ - # TODO + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_inverse_regex_match.sieve') + + # positive test (key mismatch) + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'http://www.example.com' + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # positive test (key undefined) + event = EXAMPLE_INPUT.copy() + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test (key match) + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'https://www.switch.ch/security' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) def test_numeric_equal_match(self): """ Test == numeric match """ @@ -416,13 +554,14 @@ def test_not_exists_match(self): def test_string_match_value_list(self): """ Test string match with StringValueList """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_string_match_value_list.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_match_value_list.sieve') # Match the first rule string_value_list_match_1 = EXAMPLE_INPUT.copy() string_value_list_match_1['classification.type'] = 'malware' - string_value_list_expected_result_1=string_value_list_match_1.copy() - string_value_list_expected_result_1['comment'] ='infected hosts' + string_value_list_expected_result_1 = string_value_list_match_1.copy() + string_value_list_expected_result_1['comment'] = 'infected hosts' self.input_message = string_value_list_match_1 self.run_bot() self.assertMessageEqual(0, string_value_list_expected_result_1) @@ -430,23 +569,23 @@ def test_string_match_value_list(self): # Match the second rule string_value_list_match_2 = EXAMPLE_INPUT.copy() string_value_list_match_2['classification.type'] = 'c&c' - string_value_list_expected_result_2=string_value_list_match_2.copy() - string_value_list_expected_result_2['comment'] ='malicious server / service' + string_value_list_expected_result_2 = string_value_list_match_2.copy() + string_value_list_expected_result_2['comment'] = 'malicious server / service' self.input_message = string_value_list_match_2 self.run_bot() self.assertMessageEqual(0, string_value_list_expected_result_2) - #don't Match any rule + # don't match any rule string_value_list_match_3 = EXAMPLE_INPUT.copy() string_value_list_match_3['classification.type'] = 'blacklist' self.input_message = string_value_list_match_3 self.run_bot() self.assertMessageEqual(0, string_value_list_match_3) - def test_numeric_match_value_list(self): """ Test numeric match with NumericValueList """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_numeric_match_value_list.sieve') + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_match_value_list.sieve') # Match the first rule numeric_value_list_match_1 = EXAMPLE_INPUT.copy() diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve new file mode 100644 index 000000000..0bf780499 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve @@ -0,0 +1,7 @@ +if source.abuse_contact == 'abuse@example.com' && source.ip == '1.2.3.4' || feed.provider == 'acme' { + add comment = 'match1' +} + +if source.abuse_contact == 'abuse@example.com' && (source.ip == '5.6.7.8' || feed.provider == 'acme') { + add comment = 'match2' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve new file mode 100644 index 000000000..2ec5741f1 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve @@ -0,0 +1,3 @@ +if source.url :contains '.ch' { + add comment = 'match' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve new file mode 100644 index 000000000..7168b3dae --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve @@ -0,0 +1,3 @@ +if source.fqdn == 'www.example.com' { + add comment = 'match' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve new file mode 100644 index 000000000..540804396 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve @@ -0,0 +1,3 @@ +if source.url !~ '^http(s)?://www\.switch\.ch/.*' { + add comment = 'match' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve new file mode 100644 index 000000000..c371c27bb --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve @@ -0,0 +1,3 @@ +if source.fqdn != 'www.example.com' { + add comment = 'match' +} \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve new file mode 100644 index 000000000..15f901600 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve @@ -0,0 +1,3 @@ +if source.url =~ '^http(s)?://www\.switch\.ch/.*' { + add comment = 'match' +} \ No newline at end of file From 8f1881c9a1becc9e2294d1383b20691ba840e49b Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 25 Aug 2017 12:52:06 +0200 Subject: [PATCH 20/40] Added REQUIREMENTS.txt with depedencies, antoinet/intelmq#13 --- intelmq/bots/experts/sieve/README.md | 7 +++++++ intelmq/bots/experts/sieve/REQUIREMENTS.txt | 2 ++ setup.py | 1 - 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 intelmq/bots/experts/sieve/REQUIREMENTS.txt diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index a89f9ab7d..f90e52ffa 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -157,3 +157,10 @@ positional arguments: optional arguments: -h, --help show this help message and exit ``` + +## Installation + +To use this bot, you need to install the required dependencies: +``` +$ pip install -r REQUIREMENTS.txt +``` \ No newline at end of file diff --git a/intelmq/bots/experts/sieve/REQUIREMENTS.txt b/intelmq/bots/experts/sieve/REQUIREMENTS.txt new file mode 100644 index 000000000..dcdf1bd36 --- /dev/null +++ b/intelmq/bots/experts/sieve/REQUIREMENTS.txt @@ -0,0 +1,2 @@ +textX>=1.5.1 +ipaddress>=1.0.18 diff --git a/setup.py b/setup.py index f40057a01..b7262ef84 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ 'python-termstyle>=0.1.10', 'pytz>=2014.1', 'redis>=2.10.3', - 'requests>=2.2.0' ] if sys.version_info < (3, 5): REQUIRES.append('typing') From 17c0d4d1d70903ff08f00a33ab6cb1a3276dc220 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Mon, 28 Aug 2017 10:24:25 +0200 Subject: [PATCH 21/40] Fixed codestyle errors. See: https://travis-ci.org/certtools/intelmq/jobs/268325253#L3166 --- intelmq/bots/experts/sieve/expert.py | 1 + intelmq/bots/experts/sieve/validator.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 3704e9bb4..905064ea6 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -219,4 +219,5 @@ def validate_ip_range(ip_range): except ValueError: raise TextXSemanticError('Invalid ip range: %s.', ip_range.value) + BOT = SieveExpertBot diff --git a/intelmq/bots/experts/sieve/validator.py b/intelmq/bots/experts/sieve/validator.py index 0f840edc6..59bbd1109 100644 --- a/intelmq/bots/experts/sieve/validator.py +++ b/intelmq/bots/experts/sieve/validator.py @@ -43,6 +43,7 @@ def parse(self, filename): self.logger.info('Sieve file %r parsed successfully.', filename) + if __name__ == "__main__": logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) parser = argparse.ArgumentParser(description="Validates the syntax of sievebot files.") @@ -51,4 +52,4 @@ def parse(self, filename): args = parser.parse_args() validator = Validator() - validator.parse(args.sievefile) \ No newline at end of file + validator.parse(args.sievefile) From 1176a85f9ec9e02e23b5374b5ebdfd65c01e2f4f Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Mon, 28 Aug 2017 10:55:23 +0200 Subject: [PATCH 22/40] Fixed first PR review outcomes, antoinet/intelmq#14 See: https://github.com/certtools/intelmq/pull/1083#pullrequestreview-58881709 * removed tabs in BOTS * safely import textx package * added `skip_exotic` decorator to sievebot test class * removed package `ipaddress` from REQUIREMENTS.txt Other changes: * removed `print` statement from expert.py --- intelmq/bots/BOTS | 4 ++-- intelmq/bots/experts/sieve/REQUIREMENTS.txt | 1 - intelmq/bots/experts/sieve/expert.py | 13 ++++++++++--- intelmq/tests/bots/experts/sieve/test_expert.py | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index 2ab63a066..f6393cd66 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -572,11 +572,11 @@ "module": "intelmq.bots.experts.taxonomy.expert", "parameters": {} }, - "Sievebot": { + "Sievebot": { "description": "Sievebot is the bot responsible to filter and modify intelMQ events.", "module": "intelmq.bots.experts.sieve.expert", "parameters": { - "file" :"/opt/intelmq/var/lib/bots/filter.sieve" + "file" :"/opt/intelmq/var/lib/bots/filter.sieve" } }, "Tor Nodes": { diff --git a/intelmq/bots/experts/sieve/REQUIREMENTS.txt b/intelmq/bots/experts/sieve/REQUIREMENTS.txt index dcdf1bd36..dbf5d998d 100644 --- a/intelmq/bots/experts/sieve/REQUIREMENTS.txt +++ b/intelmq/bots/experts/sieve/REQUIREMENTS.txt @@ -1,2 +1 @@ textX>=1.5.1 -ipaddress>=1.0.18 diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 905064ea6..511e92e8e 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -14,8 +14,11 @@ import ipaddress from enum import Enum from intelmq.lib.bot import Bot -from textx.metamodel import metamodel_from_file -from textx.exceptions import TextXError, TextXSemanticError + +try: + import textx +except ImportError: + textx = None class Procedure(Enum): @@ -27,6 +30,11 @@ class Procedure(Enum): class SieveExpertBot(Bot): def init(self): + if textx is None: + raise ValueError('Could not import textx. Please install it.') + from textx.metamodel import metamodel_from_file + from textx.exceptions import TextXError, TextXSemanticError + # read the sieve grammar try: filename = os.path.join(os.path.dirname(__file__), 'sieve.tx') @@ -189,7 +197,6 @@ def process_ip_range_match(self, key, ip_range, event): return False def process_action(self, action, event): - print(type(event)) if action == 'drop': return Procedure.DROP elif action == 'keep': diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index dc90c5498..b80c5f61a 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -13,6 +13,7 @@ } +@test.skip_exotic() class TestSieveExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for SieveExpertBot. From cda7c7dcfdcd25211c74a2d60f3f3f743f8ea677 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Mon, 28 Aug 2017 11:51:37 +0200 Subject: [PATCH 23/40] Fix failing tests, antoinet/intelmq#14 * Fixed BOTS format * replaced re.fullsearch with re.search (method unavailable in python < 3.4) * removed Enum definition (unavailable in python < 3.4) --- intelmq/bots/BOTS | 4 ++-- intelmq/bots/experts/sieve/expert.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index f6393cd66..abac9f2e7 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -572,11 +572,11 @@ "module": "intelmq.bots.experts.taxonomy.expert", "parameters": {} }, - "Sievebot": { + "Sievebot": { "description": "Sievebot is the bot responsible to filter and modify intelMQ events.", "module": "intelmq.bots.experts.sieve.expert", "parameters": { - "file" :"/opt/intelmq/var/lib/bots/filter.sieve" + "file": "/opt/intelmq/var/lib/bots/sieve/filter.sieve" } }, "Tor Nodes": { diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 511e92e8e..3f347b66d 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -12,7 +12,6 @@ import intelmq.lib.exceptions as exceptions import re import ipaddress -from enum import Enum from intelmq.lib.bot import Bot try: @@ -21,7 +20,7 @@ textx = None -class Procedure(Enum): +class Procedure: CONTINUE = 1 # continue processing subsequent rules (default) KEEP = 2 # stop processing and keep event DROP = 3 # stop processing and drop event @@ -157,9 +156,9 @@ def process_string_operator(self, lhs, op, rhs): elif op == ':contains': return lhs.find(rhs) >= 0 elif op == '=~': - return re.fullmatch(rhs, lhs) is not None + return re.search(rhs, lhs) is not None elif op == '!~': - return re.fullmatch(rhs, lhs) is None + return re.search(rhs, lhs) is None def process_numeric_match(self, key, op, value, event): if key not in event: From 3095d3466d4ddb45c87dcfbf14537b62c742f4fd Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 28 Aug 2017 16:25:37 +0200 Subject: [PATCH 24/40] DOC: add sieve filter expert to Bots.md --- docs/Bots.md | 17 +++++++++++++++++ intelmq/bots/BOTS | 4 ++-- intelmq/bots/experts/sieve/README.md | 4 ++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/Bots.md b/docs/Bots.md index 8670e9165..2d16995bc 100644 --- a/docs/Bots.md +++ b/docs/Bots.md @@ -721,6 +721,23 @@ Sources: * * * +### Sieve + +See intelmq/bots/experts/sieve/README.md + +#### Information: +* `name:` sieve +* `lookup:` none +* `public:` yes +* `cache (redis db):` none +* `description:` Filtering with a sieve-based configuration language + +#### Configuration Parameters: + +* `file`: Path to sieve file. Syntax can be validated with `intelmq_sieve_expert_validator`. + +* * * + ### Taxonomy #### Information: diff --git a/intelmq/bots/BOTS b/intelmq/bots/BOTS index abac9f2e7..8994459cd 100644 --- a/intelmq/bots/BOTS +++ b/intelmq/bots/BOTS @@ -572,8 +572,8 @@ "module": "intelmq.bots.experts.taxonomy.expert", "parameters": {} }, - "Sievebot": { - "description": "Sievebot is the bot responsible to filter and modify intelMQ events.", + "Sieve": { + "description": "This bot filters and modifies events based on a sieve-based language.", "module": "intelmq.bots.experts.sieve.expert", "parameters": { "file": "/opt/intelmq/var/lib/bots/sieve/filter.sieve" diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index f90e52ffa..a09a30612 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -44,7 +44,7 @@ if classification.type == ['phishing', 'malware'] && source.fqdn =~ '.*\.(ch|li) ## Parameters -The sieve bot only takes one parameter: +The sieve bot takes only one parameter: * `file` - filesystem path the the sieve file @@ -163,4 +163,4 @@ optional arguments: To use this bot, you need to install the required dependencies: ``` $ pip install -r REQUIREMENTS.txt -``` \ No newline at end of file +``` From 61b4d0c6b33e6caef250b28360a52864bbaaeb07 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 28 Aug 2017 16:27:07 +0200 Subject: [PATCH 25/40] ENH: sieve filter expert validator is executable --- intelmq/bots/experts/sieve/README.md | 4 ++-- intelmq/bots/experts/sieve/validator.py | 6 ++++-- setup.py | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index a09a30612..f242df56d 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -146,8 +146,8 @@ Comments may be used in the sieve file: all characters after `//` and until the Use the following command to validate your sieve files: ``` -$ python intelmq/bots/experts/sieve/validator.py -h -usage: validator.py [-h] sievefile +$ intelmq.bots.experts.sieve.validator +usage: intelmq.bots.experts.sieve.validator [-h] sievefile Validates the syntax of sievebot files. diff --git a/intelmq/bots/experts/sieve/validator.py b/intelmq/bots/experts/sieve/validator.py index 59bbd1109..158970492 100644 --- a/intelmq/bots/experts/sieve/validator.py +++ b/intelmq/bots/experts/sieve/validator.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os @@ -44,7 +43,7 @@ def parse(self, filename): self.logger.info('Sieve file %r parsed successfully.', filename) -if __name__ == "__main__": +def main(): # pragma: nocover logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) parser = argparse.ArgumentParser(description="Validates the syntax of sievebot files.") parser.add_argument('sievefile', help='Sieve file') @@ -53,3 +52,6 @@ def parse(self, filename): validator = Validator() validator.parse(args.sievefile) + +if __name__ == "__main__": # pragma: nocover + main() diff --git a/setup.py b/setup.py index b7262ef84..e0251f6f8 100644 --- a/setup.py +++ b/setup.py @@ -96,6 +96,7 @@ 'intelmqctl = intelmq.bin.intelmqctl:main', 'intelmqdump = intelmq.bin.intelmqdump:main', 'intelmq_psql_initdb = intelmq.bin.intelmq_psql_initdb:main', + 'intelmq.bots.experts.sieve.validator = intelmq.bots.experts.sieve.validator:main', ] + BOTS, }, scripts=[ From 6d7bca00082609725ca297d62e9fe7b781f86dd8 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 28 Aug 2017 16:33:12 +0200 Subject: [PATCH 26/40] MAINT: sieve filter expert: imports --- intelmq/bots/experts/sieve/expert.py | 17 +++++++---------- intelmq/bots/experts/sieve/validator.py | 6 +++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 3f347b66d..d17d964b8 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -5,19 +5,18 @@ Parameters: file: string """ -from __future__ import unicode_literals - -# imports for additional libraries and intelmq +import ipaddress import os -import intelmq.lib.exceptions as exceptions import re -import ipaddress + +import intelmq.lib.exceptions as exceptions from intelmq.lib.bot import Bot try: - import textx + from textx.metamodel import metamodel_from_file + from textx.exceptions import TextXError, TextXSemanticError except ImportError: - textx = None + metamodel_from_file = None class Procedure: @@ -29,10 +28,8 @@ class Procedure: class SieveExpertBot(Bot): def init(self): - if textx is None: + if metamodel_from_file is None: raise ValueError('Could not import textx. Please install it.') - from textx.metamodel import metamodel_from_file - from textx.exceptions import TextXError, TextXSemanticError # read the sieve grammar try: diff --git a/intelmq/bots/experts/sieve/validator.py b/intelmq/bots/experts/sieve/validator.py index 158970492..e4ec92e24 100644 --- a/intelmq/bots/experts/sieve/validator.py +++ b/intelmq/bots/experts/sieve/validator.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- - -import os import argparse import logging -from textx.metamodel import metamodel_from_file +import os + from textx.exceptions import TextXError +from textx.metamodel import metamodel_from_file class Validator(object): From ea03ee2ef3250b470a788b6759f0df3937ae573f Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 28 Aug 2017 16:43:10 +0200 Subject: [PATCH 27/40] BUG: sieve filter expert error handling --- intelmq/bots/experts/sieve/expert.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index d17d964b8..bf596e787 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -37,9 +37,7 @@ def init(self): self.metamodel = metamodel_from_file(filename) self.metamodel.register_obj_processors({'SingleIpRange': SieveExpertBot.validate_ip_range}) except TextXError as e: - self.logger.error('Could not process sieve grammar file. Error in (%d, %d).', e.line, e.col) - self.logger.error(str(e)) - self.stop() + raise ValueError('Could not process sieve grammar file. Error in (%d, %d): %s' % (e.line, e.col, str(e))) # validate parameters if not os.path.exists(self.parameters.file): @@ -49,9 +47,7 @@ def init(self): try: self.sieve = self.metamodel.model_from_file(self.parameters.file) except TextXError as e: - self.logger.error('Could not parse sieve file %r, error in (%d, %d).', self.parameters.file, e.line, e.col) - self.logger.error(str(e)) - self.stop() + raise ValueError('Could not parse sieve file %r, error in (%d, %d): %s' % (self.parameters.file, e.line, e.col, str(e))) def process(self): event = self.receive_message() From 8276b5ea6eb2c7f5db8d85d1867886737d3be2dc Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 28 Aug 2017 16:52:19 +0200 Subject: [PATCH 28/40] MAINT: sieve expert bot: shorten some code --- intelmq/bots/experts/sieve/expert.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index bf596e787..a122d5569 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -121,7 +121,6 @@ def process_condition(self, cond, event): return self.process_ip_range_match(match.key, match.range, event) elif match.__class__.__name__ == 'Expression': return self.match_expression(match, event) - pass def process_exist_match(self, key, op, event): if op == ':exists': @@ -208,8 +207,7 @@ def process_action(self, action, event): def get_position(self, entity): """ returns the position (line,col) of an entity in the sieve file. """ - parser = self.metamodel.parser - return parser.pos_to_linecol(entity._tx_position) + return self.metamodel.parser.pos_to_linecol(entity._tx_position) @staticmethod def validate_ip_range(ip_range): From 6d50608f50f374f26cb48bcbc8b89fdfa24f0f34 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 28 Aug 2017 17:10:59 +0200 Subject: [PATCH 29/40] TST: sieve filter expert: add missing newlines --- .../tests/bots/experts/sieve/test_sieve_files/test_add.sieve | 2 +- .../bots/experts/sieve/test_sieve_files/test_add_force.sieve | 2 +- .../bots/experts/sieve/test_sieve_files/test_drop_event.sieve | 2 +- .../bots/experts/sieve/test_sieve_files/test_exists_match.sieve | 2 +- .../bots/experts/sieve/test_sieve_files/test_if_clause.sieve | 2 -- .../experts/sieve/test_sieve_files/test_if_elif_clause.sieve | 2 +- .../sieve/test_sieve_files/test_if_elif_else_clause.sieve | 2 +- .../experts/sieve/test_sieve_files/test_if_else_clause.sieve | 2 +- .../sieve/test_sieve_files/test_ip_range_list_match.sieve | 2 +- .../experts/sieve/test_sieve_files/test_ip_range_match.sieve | 2 +- .../bots/experts/sieve/test_sieve_files/test_keep_event.sieve | 2 +- .../tests/bots/experts/sieve/test_sieve_files/test_modify.sieve | 2 +- .../experts/sieve/test_sieve_files/test_multiple_actions.sieve | 2 +- .../experts/sieve/test_sieve_files/test_notexists_match.sieve | 2 +- .../sieve/test_sieve_files/test_numeric_equal_match.sieve | 2 +- .../test_numeric_greater_than_or_equal_match.sieve | 2 +- .../test_numeric_less_than_or_equal_match.sieve | 2 +- .../sieve/test_sieve_files/test_numeric_match_value_list.sieve | 2 +- .../sieve/test_sieve_files/test_numeric_not_equal_match.sieve | 2 +- .../bots/experts/sieve/test_sieve_files/test_precedence.sieve | 2 +- .../tests/bots/experts/sieve/test_sieve_files/test_remove.sieve | 2 +- .../sieve/test_sieve_files/test_string_contains_match.sieve | 2 +- .../sieve/test_sieve_files/test_string_equal_match.sieve | 2 +- .../test_sieve_files/test_string_inverse_regex_match.sieve | 2 +- .../sieve/test_sieve_files/test_string_match_value_list.sieve | 2 +- .../sieve/test_sieve_files/test_string_not_equal_match.sieve | 2 +- .../sieve/test_sieve_files/test_string_regex_match.sieve | 2 +- 27 files changed, 26 insertions(+), 28 deletions(-) diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve index bc7dbdcc3..856ec124f 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add.sieve @@ -1,3 +1,3 @@ if comment == 'add field' { add destination.ip="150.50.50.10" -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve index 45939f238..d1935da09 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_add_force.sieve @@ -4,4 +4,4 @@ if comment == 'add force new field' { if comment == 'add force existing fields' { add! destination.ip = "200.10.9.7" add! source.ip = "10.9.8.7" -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve index 023372a07..8aa74223e 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve @@ -4,4 +4,4 @@ if comment == 'drop' { if source.ip == '127.0.0.1' { modify comment = 'changed' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve index fc070c292..9cc79900f 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_exists_match.sieve @@ -1,3 +1,3 @@ if :exists source.fqdn { add comment = "I think therefore I am." -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve index df307ee85..ad90e95be 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve @@ -5,5 +5,3 @@ if comment == 'changeme' { if source.ip == '192.168.0.1' { modify source.ip = '192.168.0.2' } - - diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve index 61f9ac8ec..25cc4df1b 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve @@ -4,4 +4,4 @@ if comment == 'match1' { modify comment = 'changed2' } elif comment == 'match3' { modify comment = 'changed3' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve index fe8b0d36c..ec1c70684 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve @@ -4,4 +4,4 @@ if comment == 'match1' { modify comment = 'changed2' } else { modify comment = 'changed3' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve index a3bfa21d5..e257a94cf 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve @@ -2,4 +2,4 @@ if comment == 'match' { modify comment = 'matched' } else { modify comment = 'notmatched' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve index 418582309..a0ab926bb 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_list_match.sieve @@ -1,3 +1,3 @@ if source.ip << ['192.0.0.0/24', '169.254.0.0/16'] { add comment = 'bogon' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve index 2de1c457d..b8583c9e0 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_ip_range_match.sieve @@ -15,4 +15,4 @@ if source.ip << '2001:620:0:0:0:0:0:0/29' { if comment << '10.0.0.0/8' { add comment = 'valid' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve index 932f37017..d206a719a 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve @@ -7,4 +7,4 @@ if comment == 'keep' { if comment == ['continue', 'keep'] { modify comment = 'changed' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve index 11b1cfa02..d5b5c329c 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve @@ -3,4 +3,4 @@ if comment == 'modify new parameter' { } if comment == 'modify existing parameter' { modify source.ip="10.9.8.7" -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve index 605e1b694..5d77b34ab 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve @@ -2,4 +2,4 @@ if :exists source.ip { add comment = 'added' modify source.ip = '127.0.0.2' remove classification.type -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve index 794dadc62..f93d35ca3 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_notexists_match.sieve @@ -1,3 +1,3 @@ if :notexists source.fqdn { add comment = "I think therefore I am." -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve index 27109ea08..3d6ff882e 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_equal_match.sieve @@ -1 +1 @@ -if feed.accuracy == 100.0 { drop } \ No newline at end of file +if feed.accuracy == 100.0 { drop } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve index 294f57655..4385b367b 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_greater_than_or_equal_match.sieve @@ -1 +1 @@ -if feed.accuracy >= 90 { drop } \ No newline at end of file +if feed.accuracy >= 90 { drop } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve index beeac6cb9..7222101b5 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_less_than_or_equal_match.sieve @@ -1 +1 @@ -if feed.accuracy <= 90 { drop } \ No newline at end of file +if feed.accuracy <= 90 { drop } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve index 765ca934e..40557bb90 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_match_value_list.sieve @@ -8,4 +8,4 @@ if destination.asn == [1930,199155] { add comment='Belongs constituency group' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve index 0c61006a5..a2cb8b296 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_not_equal_match.sieve @@ -1 +1 @@ -if feed.accuracy != 100.0 { drop } \ No newline at end of file +if feed.accuracy != 100.0 { drop } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve index 0bf780499..2ea8ae4c4 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_precedence.sieve @@ -4,4 +4,4 @@ if source.abuse_contact == 'abuse@example.com' && source.ip == '1.2.3.4' || feed if source.abuse_contact == 'abuse@example.com' && (source.ip == '5.6.7.8' || feed.provider == 'acme') { add comment = 'match2' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve index f59658956..d25f51b3b 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_remove.sieve @@ -1,3 +1,3 @@ if comment == 'remove parameter' { remove destination.ip -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve index 2ec5741f1..ba02a5d6a 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_contains_match.sieve @@ -1,3 +1,3 @@ if source.url :contains '.ch' { add comment = 'match' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve index 7168b3dae..3d9ce6059 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_equal_match.sieve @@ -1,3 +1,3 @@ if source.fqdn == 'www.example.com' { add comment = 'match' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve index 540804396..bfc346f50 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_inverse_regex_match.sieve @@ -1,3 +1,3 @@ if source.url !~ '^http(s)?://www\.switch\.ch/.*' { add comment = 'match' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve index c75877367..8f40a1796 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_match_value_list.sieve @@ -8,4 +8,4 @@ if classification.type == ['c&c','malware configuration'] { add comment='malicious server / service' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve index c371c27bb..78c2e7ef8 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_not_equal_match.sieve @@ -1,3 +1,3 @@ if source.fqdn != 'www.example.com' { add comment = 'match' -} \ No newline at end of file +} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve index 15f901600..0d6525226 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_regex_match.sieve @@ -1,3 +1,3 @@ if source.url =~ '^http(s)?://www\.switch\.ch/.*' { add comment = 'match' -} \ No newline at end of file +} From f590a95c98eba4650949447caa02bba8fe5c0e3a Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Thu, 31 Aug 2017 00:51:41 +0200 Subject: [PATCH 30/40] Review quick fixes, antoinet/intelmq#14 - Remove .gitignore for .dot files - Typo and erroneous syntax in README.md - codestyle in validator.py --- intelmq/bots/experts/sieve/.gitignore | 2 -- intelmq/bots/experts/sieve/README.md | 4 ++-- intelmq/bots/experts/sieve/validator.py | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 intelmq/bots/experts/sieve/.gitignore diff --git a/intelmq/bots/experts/sieve/.gitignore b/intelmq/bots/experts/sieve/.gitignore deleted file mode 100644 index 3be615847..000000000 --- a/intelmq/bots/experts/sieve/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.dot - diff --git a/intelmq/bots/experts/sieve/README.md b/intelmq/bots/experts/sieve/README.md index f242df56d..3f68697e5 100644 --- a/intelmq/bots/experts/sieve/README.md +++ b/intelmq/bots/experts/sieve/README.md @@ -26,7 +26,7 @@ if :notexists source.abuse_contact || source.abuse_contact =~ '.*@example.com' { drop // aborts processing of subsequent rules and drops the event. } -if source.ip << 192.0.0.0/24 { +if source.ip << '192.0.0.0/24' { add! comment = 'bogon' } @@ -45,7 +45,7 @@ if classification.type == ['phishing', 'malware'] && source.fqdn =~ '.*\.(ch|li) ## Parameters The sieve bot takes only one parameter: - * `file` - filesystem path the the sieve file + * `file` - filesystem path of the sieve file ## Reference diff --git a/intelmq/bots/experts/sieve/validator.py b/intelmq/bots/experts/sieve/validator.py index e4ec92e24..d10a250b1 100644 --- a/intelmq/bots/experts/sieve/validator.py +++ b/intelmq/bots/experts/sieve/validator.py @@ -53,5 +53,6 @@ def main(): # pragma: nocover validator = Validator() validator.parse(args.sievefile) + if __name__ == "__main__": # pragma: nocover main() From b46b6428ccd9ff0e0b46563aeba640a98af438cc Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Thu, 7 Sep 2017 11:43:22 +0200 Subject: [PATCH 31/40] antoinet/intelmq#18: rename action `modify` to `update`. --- intelmq/bots/experts/sieve/expert.py | 2 +- intelmq/bots/experts/sieve/sieve.tx | 4 ++-- intelmq/tests/bots/experts/sieve/test_expert.py | 10 +++++----- .../sieve/test_sieve_files/test_drop_event.sieve | 2 +- .../sieve/test_sieve_files/test_if_clause.sieve | 4 ++-- .../sieve/test_sieve_files/test_if_elif_clause.sieve | 6 +++--- .../test_sieve_files/test_if_elif_else_clause.sieve | 6 +++--- .../sieve/test_sieve_files/test_if_else_clause.sieve | 4 ++-- .../sieve/test_sieve_files/test_keep_event.sieve | 2 +- .../experts/sieve/test_sieve_files/test_modify.sieve | 6 ------ .../sieve/test_sieve_files/test_multiple_actions.sieve | 2 +- .../experts/sieve/test_sieve_files/test_or_match.sieve | 2 +- .../experts/sieve/test_sieve_files/test_update.sieve | 6 ++++++ 13 files changed, 28 insertions(+), 28 deletions(-) delete mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_update.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index a122d5569..198947c3e 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -197,7 +197,7 @@ def process_action(self, action, event): event.add(action.key, action.value) elif action.__class__.__name__ == 'AddForceAction': event.add(action.key, action.value, overwrite=True) - elif action.__class__.__name__ == 'ModifyAction': + elif action.__class__.__name__ == 'UpdateAction': if action.key in event: event.change(action.key, action.value) elif action.__class__.__name__ == 'RemoveAction': diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index 7fd8ce444..f6cb5cc69 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -63,14 +63,14 @@ Action: action=DropAction | ( action=KeepAction | action=AddAction | action=AddForceAction - | action=ModifyAction + | action=UpdateAction | action=RemoveAction ); DropAction: 'drop'; KeepAction: 'keep'; AddAction: 'add' key=Key '=' value=STRING; // add key/value without overwriting existing key AddForceAction: 'add!' key=Key '=' value=STRING; // add key/value, overwriting existing key -ModifyAction: 'modify' key=Key '=' value=STRING; // modify key/value, do not create if not exists +UpdateAction: 'update' key=Key '=' value=STRING; // update key/value, do not create if not exists RemoveAction: 'remove' key=Key; Comment: /\/\/.*$/ ; \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index b80c5f61a..0890f50b0 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -694,9 +694,9 @@ def test_add_force(self): self.run_bot() self.assertMessageEqual(0, result2) - def test_modify(self): - """ Test modifying key/value pairs """ - self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_modify.sieve') + def test_update(self): + """ Test updating key/value pairs """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_update.sieve') # If doesn't match, nothing should have changed event1 = EXAMPLE_INPUT.copy() @@ -705,7 +705,7 @@ def test_modify(self): self.assertMessageEqual(0, event1) # If expression matches && parameter doesn't exists, nothing changes - event1['comment'] = 'modify new parameter' + event1['comment'] = 'update new parameter' result = event1.copy() self.input_message = event1 self.run_bot() @@ -713,7 +713,7 @@ def test_modify(self): # If expression matches && parameter exists, source.ip changed event2 = EXAMPLE_INPUT.copy() - event2['comment'] = 'modify existing parameter' + event2['comment'] = 'update existing parameter' result2 = event2.copy() result2['source.ip'] = '10.9.8.7' self.input_message = event2 diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve index 8aa74223e..a5e172afc 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_drop_event.sieve @@ -3,5 +3,5 @@ if comment == 'drop' { } if source.ip == '127.0.0.1' { - modify comment = 'changed' + update comment = 'changed' } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve index ad90e95be..587e96ac4 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_clause.sieve @@ -1,7 +1,7 @@ if comment == 'changeme' { - modify comment = 'changed' + update comment = 'changed' } if source.ip == '192.168.0.1' { - modify source.ip = '192.168.0.2' + update source.ip = '192.168.0.2' } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve index 25cc4df1b..b6c8843a9 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_clause.sieve @@ -1,7 +1,7 @@ if comment == 'match1' { - modify comment = 'changed1' + update comment = 'changed1' } elif comment == 'match2' { - modify comment = 'changed2' + update comment = 'changed2' } elif comment == 'match3' { - modify comment = 'changed3' + update comment = 'changed3' } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve index ec1c70684..85a0afbdf 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_elif_else_clause.sieve @@ -1,7 +1,7 @@ if comment == 'match1' { - modify comment = 'changed1' + update comment = 'changed1' } elif comment == 'match2' { - modify comment = 'changed2' + update comment = 'changed2' } else { - modify comment = 'changed3' + update comment = 'changed3' } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve index e257a94cf..51fdef337 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_if_else_clause.sieve @@ -1,5 +1,5 @@ if comment == 'match' { - modify comment = 'matched' + update comment = 'matched' } else { - modify comment = 'notmatched' + update comment = 'notmatched' } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve index d206a719a..40564f3c7 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_keep_event.sieve @@ -6,5 +6,5 @@ if comment == 'keep' { } if comment == ['continue', 'keep'] { - modify comment = 'changed' + update comment = 'changed' } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve deleted file mode 100644 index d5b5c329c..000000000 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_modify.sieve +++ /dev/null @@ -1,6 +0,0 @@ -if comment == 'modify new parameter' { - modify destination.ip="100.99.88.77" -} -if comment == 'modify existing parameter' { - modify source.ip="10.9.8.7" -} diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve index 5d77b34ab..5b94dc669 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_multiple_actions.sieve @@ -1,5 +1,5 @@ if :exists source.ip { add comment = 'added' - modify source.ip = '127.0.0.2' + update source.ip = '127.0.0.2' remove classification.type } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve index 08d9c8470..008009982 100644 --- a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_or_match.sieve @@ -1,3 +1,3 @@ if ( source.abuse_contact == "abuse@example.com" || comment == "I am TRUE in OR clause" ) { - modify source.ip = "10.9.8.7" + update source.ip = "10.9.8.7" } diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_update.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_update.sieve new file mode 100644 index 000000000..daf3c8094 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_update.sieve @@ -0,0 +1,6 @@ +if comment == 'update new parameter' { + update destination.ip="100.99.88.77" +} +if comment == 'update existing parameter' { + update source.ip="10.9.8.7" +} From e385bfa0f6e01b5597fe37d83a3639aa0a05a1c8 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Thu, 7 Sep 2017 11:44:49 +0200 Subject: [PATCH 32/40] Re-enable cymru-whois tests (were disabled because service was temporarily unavailable). --- intelmq/tests/bots/experts/cymru_whois/test_expert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/intelmq/tests/bots/experts/cymru_whois/test_expert.py b/intelmq/tests/bots/experts/cymru_whois/test_expert.py index 5b2e9718e..bb2504d6a 100644 --- a/intelmq/tests/bots/experts/cymru_whois/test_expert.py +++ b/intelmq/tests/bots/experts/cymru_whois/test_expert.py @@ -78,7 +78,6 @@ @test.skip_redis() @test.skip_internet() -@unittest.skip('cymru is currently unreachable') class TestCymruExpertBot(test.BotTestCase, unittest.TestCase): """ A TestCase for AbusixExpertBot. From 2d5ac01ca239974979c50a8158a83648816f3928 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 11 Sep 2017 13:02:09 +0200 Subject: [PATCH 33/40] ENH+DOC: check function for bots fixes certtools/intelmq#1090 --- CHANGELOG.md | 2 ++ docs/Developers-Guide.md | 17 +++++++++++++++++ intelmq/bin/intelmqctl.py | 13 ++++++++++++- intelmq/lib/bot.py | 19 ++++++++++++++++++- 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 514799848..eb41aac2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ CHANGELOG event['extra'] # gives '{"foo": "bar"}' "Old" bots and configurations compatible with 1.0.x do still work. Also, the extra field is now properly exploded when exporting events, analogous to all other fields. +- Bots can specify a static method `check(parameters)` which can perform individual checks specific to the bot. + These functions will be called by `intelmqctl check` if the bot is configured with the given parameters ### Bots #### Collectors diff --git a/docs/Developers-Guide.md b/docs/Developers-Guide.md index 4733f3860..0b00b7557 100644 --- a/docs/Developers-Guide.md +++ b/docs/Developers-Guide.md @@ -37,6 +37,7 @@ * [How to Log](#how-to-log) * [Error handling](#error-handling) * [Initialization](#initialization) + * [Custom configuration checks](#custom-configuration-checks) * [Examples](#examples) * [Parsers](#parsers) * [Tests](#tests) @@ -481,6 +482,22 @@ class ExampleParserBot(Bot): self.stop() ``` +## Custom configuration checks + +Every bot can define a static method `check(parameters)` which will be called by `intelmqctl check`. +For example the check function of the ASNLookupExpert: + +```python + @staticmethod + def check(parameters): + if not os.path.exists(parameters.get('database', '')): + return [["error", "File given as parameter 'database' does not exist."]] + try: + pyasn.pyasn(parameters['database']) + except Exception as exc: + return [["error", "Error reading database: %r." % exc]] +``` + ## Examples * Check [Expert Bots](../intelmq/bots/experts/) diff --git a/intelmq/bin/intelmqctl.py b/intelmq/bin/intelmqctl.py index d32100db5..2c168da16 100644 --- a/intelmq/bin/intelmqctl.py +++ b/intelmq/bin/intelmqctl.py @@ -978,13 +978,24 @@ def check(self): for bot_id, bot_config in files[RUNTIME_CONF_FILE].items(): # importable module try: - importlib.import_module(bot_config['module']) + bot_module = importlib.import_module(bot_config['module']) except ImportError: if RETURN_TYPE == 'json': output.append(['error', 'Incomplete installation: Module %r not importable.' % bot_id]) else: self.logger.error('Incomplete installation: Module %r not importable.', bot_id) retval = 1 + continue + bot = getattr(bot_module, 'BOT') + bot_parameters = files[DEFAULTS_CONF_FILE].copy() + bot_parameters.update(bot_config['parameters']) + bot_check = bot.check(bot_parameters) + if bot_check: + if RETURN_TYPE == 'json': + output.extend(bot_check) + else: + for log_line in bot_check: + getattr(self.logger, log_line[0])("Bot %r: %s" % (bot_id, log_line[1])) for group in files[BOTS_FILE].values(): for bot_id, bot in group.items(): if subprocess.call(['which', bot['module']], stdout=subprocess.DEVNULL, diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py index 90f7211fe..93b0cd5ec 100644 --- a/intelmq/lib/bot.py +++ b/intelmq/lib/bot.py @@ -21,7 +21,7 @@ from intelmq.lib import exceptions, utils import intelmq.lib.message as libmessage from intelmq.lib.pipeline import PipelineFactory -from typing import Any, Optional +from typing import Any, Optional, List __all__ = ['Bot', 'CollectorBot', 'ParserBot'] @@ -541,6 +541,23 @@ def set_request_parameters(self): self.http_header['User-agent'] = self.parameters.http_user_agent + @staticmethod + def check(parameters: dict) -> Optional[List[List[str]]]: + """ + The bot's own check function can perform individual checks on it's + parameters. + `init()` is *not* called before, this is a staticmethod which does not + require class initialization. + + Parameters: + parameters: Bot's parameters, defaults and runtime merged together + + Returns: + output: None or a list of [log_level, log_message] pairs, both + strings. log_level must be a valid log level. + """ + pass + class ParserBot(Bot): csv_params = {} From 764015b45af4c8e4b8a772ac43d7dc23d5b76962 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 11 Sep 2017 13:02:53 +0200 Subject: [PATCH 34/40] ENH: check method for asn_lookup and file output --- intelmq/bots/experts/asn_lookup/expert.py | 13 +++++++++++-- intelmq/bots/outputs/file/output.py | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/experts/asn_lookup/expert.py b/intelmq/bots/experts/asn_lookup/expert.py index 3da576a79..70a4635cb 100644 --- a/intelmq/bots/experts/asn_lookup/expert.py +++ b/intelmq/bots/experts/asn_lookup/expert.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" -""" +import os + from intelmq.lib.bot import Bot try: @@ -47,5 +47,14 @@ def process(self): self.send_message(event) self.acknowledge_message() + @staticmethod + def check(parameters): + if not os.path.exists(parameters.get('database', '')): + return [["error", "File given as parameter 'database' does not exist."]] + try: + pyasn.pyasn(parameters['database']) + except Exception as exc: + return [["error", "Error reading database: %r." % exc]] + BOT = ASNLookupExpertBot diff --git a/intelmq/bots/outputs/file/output.py b/intelmq/bots/outputs/file/output.py index 32d7fd5f1..271404a50 100644 --- a/intelmq/bots/outputs/file/output.py +++ b/intelmq/bots/outputs/file/output.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import io +import os from intelmq.lib.bot import Bot @@ -27,5 +28,13 @@ def process(self): def shutdown(self): self.file.close() + @staticmethod + def check(parameters): + if 'file' not in parameters: + return [["error", "Parameter 'file' not given."]] + dirname = os.path.dirname(parameters['file']) + if not os.path.exists(dirname): + return [["error", "Directory (%r) of parameter 'file' does not exist." % dirname]] + BOT = FileOutputBot From e878cc96fcaf0e9ae87f6ad2964299cfe3dbd3de Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 29 Sep 2017 13:06:07 +0200 Subject: [PATCH 35/40] check method for sieve filter, antoinet/intelmq#20 --- MANIFEST.in | 1 + intelmq/bots/experts/sieve/expert.py | 36 +++++++++++++++++++--------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4bf1c404b..2c87b8c78 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,4 @@ graft docs include COPYRIGHT include LICENSE include CHANGELOG.md +include intelmq/bots/experts/sieve/sieve.tx \ No newline at end of file diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 198947c3e..d18731392 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -28,26 +28,40 @@ class Procedure: class SieveExpertBot(Bot): def init(self): + self.metamodel = SieveExpertBot.init_metamodel() + self.sieve = SieveExpertBot.read_sieve_file(self.parameters.file, self.metamodel) + + @staticmethod + def init_metamodel(): if metamodel_from_file is None: - raise ValueError('Could not import textx. Please install it.') + raise ValueError('Could not import textx. Please install it') - # read the sieve grammar try: - filename = os.path.join(os.path.dirname(__file__), 'sieve.tx') - self.metamodel = metamodel_from_file(filename) - self.metamodel.register_obj_processors({'SingleIpRange': SieveExpertBot.validate_ip_range}) + grammarfile = os.path.join(os.path.dirname(__file__), 'sieve.tx') + metamodel = metamodel_from_file(grammarfile) + metamodel.register_obj_processors({'SingleIpRange': SieveExpertBot.validate_ip_range}) + return metamodel except TextXError as e: raise ValueError('Could not process sieve grammar file. Error in (%d, %d): %s' % (e.line, e.col, str(e))) - # validate parameters - if not os.path.exists(self.parameters.file): - raise exceptions.InvalidArgument('file', got=self.parameters.file, expected='existing file') + @staticmethod + def read_sieve_file(filename, metamodel): + if not os.path.exists(filename): + raise exceptions.InvalidArgument('file', got=filename, expected='existing file') - # parse sieve file try: - self.sieve = self.metamodel.model_from_file(self.parameters.file) + sieve = metamodel.model_from_file(filename) + return sieve except TextXError as e: - raise ValueError('Could not parse sieve file %r, error in (%d, %d): %s' % (self.parameters.file, e.line, e.col, str(e))) + raise ValueError('Could not parse sieve file %r, error in (%d, %d): %s' % (filename, e.line, e.col, str(e))) + + @staticmethod + def check(parameters): + try: + metamodel = SieveExpertBot.init_metamodel() + sieve = SieveExpertBot.read_sieve_file(parameters['file'], metamodel) + except Exception as e: + return [['error', str(e)]] def process(self): event = self.receive_message() From e2f6a8cc6d758b7246beb52881da664a29f89c1e Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 29 Sep 2017 13:19:45 +0200 Subject: [PATCH 36/40] Use '//' or '#' for line comments, antoinet/intelmq#19 --- intelmq/bots/experts/sieve/sieve.tx | 2 +- intelmq/tests/bots/experts/sieve/test_expert.py | 12 ++++++++++++ .../sieve/test_sieve_files/test_comments.sieve | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_comments.sieve diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index f6cb5cc69..c9f3b72d9 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -73,4 +73,4 @@ AddForceAction: 'add!' key=Key '=' value=STRING; // add key/value, overwr UpdateAction: 'update' key=Key '=' value=STRING; // update key/value, do not create if not exists RemoveAction: 'remove' key=Key; -Comment: /\/\/.*$/ ; \ No newline at end of file +Comment: /(#|\/\/).*$/ ; \ No newline at end of file diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 0890f50b0..778121ff9 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -829,6 +829,18 @@ def test_ip_range_list_match(self): self.run_bot() self.assertMessageEqual(0, event) + def test_comments(self): + """ Test comments in sieve file.""" + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_comments.sieve') + + event = EXAMPLE_INPUT.copy() + expected = event.copy() + expected['comment'] = 'hello' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_comments.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_comments.sieve new file mode 100644 index 000000000..5e28f075d --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_comments.sieve @@ -0,0 +1,5 @@ +if source.ip == '127.0.0.1' { + add comment = 'hello' + // update comment = 'I should not be here' + # update comment = 'neither should I' +} \ No newline at end of file From 342b4838d02a6cb6af81542c2bf2645c5276ce49 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Fri, 29 Sep 2017 13:32:45 +0200 Subject: [PATCH 37/40] implemented :notcontains operator, antoinet/intelmq#17 --- intelmq/bots/experts/sieve/expert.py | 4 ++- intelmq/bots/experts/sieve/sieve.tx | 11 +++---- .../tests/bots/experts/sieve/test_expert.py | 30 +++++++++++++++++++ .../test_string_notcontains_match.sieve | 3 ++ 4 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_notcontains_match.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index d18731392..253d2ad17 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -144,7 +144,7 @@ def process_exist_match(self, key, op, event): def process_string_match(self, key, op, value, event): if key not in event: - return op == '!=' or op == '!~' + return op == '!=' or op == '!~' or op == ':notcontains' if value.__class__.__name__ == 'SingleStringValue': return self.process_string_operator(event[key], op, value.value) @@ -161,6 +161,8 @@ def process_string_operator(self, lhs, op, rhs): return lhs != rhs elif op == ':contains': return lhs.find(rhs) >= 0 + elif op == ':notcontains': + return lhs.find(rhs) == -1 elif op == '=~': return re.search(rhs, lhs) is not None elif op == '!~': diff --git a/intelmq/bots/experts/sieve/sieve.tx b/intelmq/bots/experts/sieve/sieve.tx index c9f3b72d9..ffd97dc3e 100644 --- a/intelmq/bots/experts/sieve/sieve.tx +++ b/intelmq/bots/experts/sieve/sieve.tx @@ -23,11 +23,12 @@ Condition: StringMatch: key=Key op=StringOperator value=StringValue; StringOperator: - '==' // compares two whole strings with each other - | '!=' // test for string inequality - | ':contains' // sub-string match - | '=~' // match strings according to regular expression - | '!~' // inverse match with regular expression + '==' // compares two whole strings with each other + | '!=' // test for string inequality + | ':contains' // sub-string match + | ':notcontains' // inverse sub-string match + | '=~' // match strings according to regular expression + | '!~' // inverse match with regular expression ; NumericMatch: key=Key op=NumericOperator value=NumericValue; diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 778121ff9..e288e065d 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -331,6 +331,36 @@ def test_string_contains_match(self): self.run_bot() self.assertMessageEqual(0, event) + def test_string_notcontains_match(self): + """ Test :notcontains string match.""" + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_notcontains_match.sieve') + + # positive test (key undefined) + event = EXAMPLE_INPUT.copy() + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # positive test (key mismatch) + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'https://www.switch.ch/security/' + expected = event.copy() + expected['comment'] = 'match' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, expected) + + # negative test (key match) + event = EXAMPLE_INPUT.copy() + event['source.url'] = 'https://www.google.com/' + self.input_message = event + self.run_bot() + self.assertMessageEqual(0, event) + + def test_string_regex_match(self): """ Test =~ string match """ self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_notcontains_match.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_notcontains_match.sieve new file mode 100644 index 000000000..728993083 --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_notcontains_match.sieve @@ -0,0 +1,3 @@ +if source.url :notcontains '.com' { + add comment = 'match' +} From 7ee859b7425d0981d411b0a3eb565c360ae7f29c Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Thu, 5 Oct 2017 15:18:40 +0200 Subject: [PATCH 38/40] Strict checking for numeric match while using eval(), antoinet/intelmq#16 --- intelmq/bots/experts/sieve/expert.py | 77 ++++++++++++++++--- .../tests/bots/experts/sieve/test_expert.py | 12 +++ .../test_numeric_invalid_key.sieve | 3 + 3 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_invalid_key.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 253d2ad17..1c931c917 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -11,8 +11,11 @@ import intelmq.lib.exceptions as exceptions from intelmq.lib.bot import Bot +from intelmq import HARMONIZATION_CONF_FILE +from intelmq.lib import utils try: + import textx.model from textx.metamodel import metamodel_from_file from textx.exceptions import TextXError, TextXSemanticError except ImportError: @@ -27,7 +30,13 @@ class Procedure: class SieveExpertBot(Bot): + harmonization = None + def init(self): + if not SieveExpertBot.harmonization: + harmonization_config = utils.load_configuration(HARMONIZATION_CONF_FILE) + SieveExpertBot.harmonization = harmonization_config['event'] + self.metamodel = SieveExpertBot.init_metamodel() self.sieve = SieveExpertBot.read_sieve_file(self.parameters.file, self.metamodel) @@ -40,6 +49,7 @@ def init_metamodel(): grammarfile = os.path.join(os.path.dirname(__file__), 'sieve.tx') metamodel = metamodel_from_file(grammarfile) metamodel.register_obj_processors({'SingleIpRange': SieveExpertBot.validate_ip_range}) + metamodel.register_obj_processors({'NumericMatch': SieveExpertBot.validate_numeric_match}) return metamodel except TextXError as e: raise ValueError('Could not process sieve grammar file. Error in (%d, %d): %s' % (e.line, e.col, str(e))) @@ -69,10 +79,10 @@ def process(self): for rule in self.sieve.rules: procedure = self.process_rule(rule, event) if procedure == Procedure.KEEP: - self.logger.debug('Stop processing based on rule at %s: %s.', self.get_position(rule), event) + self.logger.debug('Stop processing based on rule at %s: %s.', self.get_linecol(rule), event) break elif procedure == Procedure.DROP: - self.logger.debug('Dropped event based on rule at %s: %s.', self.get_position(rule), event) + self.logger.debug('Dropped event based on rule at %s: %s.', self.get_linecol(rule), event) break # forwarding decision @@ -84,7 +94,7 @@ def process(self): def process_rule(self, rule, event): # process mandatory 'if' clause if self.match_expression(rule.if_.expr, event): - self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule.if_), event) + self.logger.debug('Matched event based on rule at %s: %s.', self.get_linecol(rule.if_), event) for action in rule.if_.actions: procedure = self.process_action(action.action, event) if procedure != Procedure.CONTINUE: @@ -94,7 +104,7 @@ def process_rule(self, rule, event): # process optional 'elif' clauses for clause in rule.elif_: if self.match_expression(clause.expr, event): - self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(clause), event) + self.logger.debug('Matched event based on rule at %s: %s.', self.get_linecol(clause), event) for action in clause.actions: procedure = self.process_action(action.action, event) if procedure != Procedure.CONTINUE: @@ -103,7 +113,7 @@ def process_rule(self, rule, event): # process optional 'else' clause if rule.else_: - self.logger.debug('Matched event based on rule at %s: %s.', self.get_position(rule.else_), event) + self.logger.debug('Matched event based on rule at %s: %s.', self.get_linecol(rule.else_), event) for action in rule.else_.actions: procedure = self.process_action(action.action, event) if procedure != Procedure.CONTINUE: @@ -181,7 +191,9 @@ def process_numeric_match(self, key, op, value, event): return False def process_numeric_operator(self, lhs, op, rhs): - return eval(str(lhs) + op + str(rhs)) # TODO graceful error handling + if not self.is_numeric(lhs) or not self.is_numeric(rhs): + return False + return eval(str(lhs) + op + str(rhs)) def process_ip_range_match(self, key, ip_range, event): if key not in event: @@ -221,16 +233,59 @@ def process_action(self, action, event): del event[action.key] return Procedure.CONTINUE - def get_position(self, entity): - """ returns the position (line,col) of an entity in the sieve file. """ - return self.metamodel.parser.pos_to_linecol(entity._tx_position) - @staticmethod def validate_ip_range(ip_range): + position = SieveExpertBot.get_linecol(ip_range, as_dict=True) try: ipaddress.ip_network(ip_range.value) except ValueError: - raise TextXSemanticError('Invalid ip range: %s.', ip_range.value) + raise TextXSemanticError('Invalid ip range: %s.' % ip_range.value, **position) + + @staticmethod + def validate_numeric_match(num_match): + """ Validates a numeric match expression. + + Checks if the event key (given on the left hand side of the expression) is of a valid type for a numeric + match, according the the IntelMQ harmonization. + + Raises: + TextXSemanticError: when the key is of an incompatible type for numeric match experessions. + """ + valid_types = ['Integer', 'Float', 'Accuracy', 'ASN'] + position = SieveExpertBot.get_linecol(num_match.value, as_dict=True) + + # validate harmonization type (event key) + try: + type = SieveExpertBot.harmonization[num_match.key]['type'] + if type not in valid_types: + raise TextXSemanticError('Incompatible type: %s.' % type, **position) + except KeyError: + raise TextXSemanticError('Invalid key: %s.' % num_match.key, **position) + + @staticmethod + def is_numeric(num): + """ Returns True if argument is a number (integer or float). """ + return str(num).lstrip('-').replace('.', '', 1).isnumeric() + + @staticmethod + def get_linecol(model_obj, as_dict=False): + """ Gets the position of a model object in the sieve file. + + Args: + model_obj: the model object + as_dict: return the position as a dict instead of a tuple. + + Returns: + Returns the line and column number for the model object's position in the sieve file. + Default return type is a tuple of (line,col). Optionally, returns a dict when as_dict == True. + + """ + metamodel = textx.model.metamodel(model_obj) + tup = metamodel.parser.pos_to_linecol(model_obj._tx_position) + if as_dict: + return dict(zip(['line', 'col'], tup)) + return tup + BOT = SieveExpertBot diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index e288e065d..2c596dd04 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -545,6 +545,18 @@ def test_numeric_greater_than_or_equal_match(self): self.run_bot() self.assertOutputQueueLen(0) + def test_numeric_invalid_key(self): + """ Tests validation of harmonization for numeric types. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_numeric_invalid_key.sieve') + + event = EXAMPLE_INPUT.copy() + self.input_message = event + with self.assertRaises(ValueError) as context: + self.run_bot() + exception = context.exception + self.assertRegex(str(exception), '.*Incompatible type: FQDN\.$') + def test_exists_match(self): """ Test :exists match """ self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), 'test_sieve_files/test_exists_match.sieve') diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_invalid_key.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_invalid_key.sieve new file mode 100644 index 000000000..b2dfe285c --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_numeric_invalid_key.sieve @@ -0,0 +1,3 @@ +if source.fqdn == 2 { + add comment = "this should fail" +} \ No newline at end of file From 1419da5dfb100d202de29c9da1894945d1bfa323 Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Thu, 5 Oct 2017 15:31:35 +0200 Subject: [PATCH 39/40] Codestyle fixes --- intelmq/bots/experts/sieve/expert.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index 1c931c917..b8991f2f9 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -146,7 +146,8 @@ def process_condition(self, cond, event): elif match.__class__.__name__ == 'Expression': return self.match_expression(match, event) - def process_exist_match(self, key, op, event): + @staticmethod + def process_exist_match(key, op, event): if op == ':exists': return key in event elif op == ':notexists': @@ -164,7 +165,8 @@ def process_string_match(self, key, op, value, event): return True return False - def process_string_operator(self, lhs, op, rhs): + @staticmethod + def process_string_operator(lhs, op, rhs): if op == '==': return lhs == rhs elif op == '!=': @@ -215,7 +217,8 @@ def process_ip_range_match(self, key, ip_range, event): return True return False - def process_action(self, action, event): + @staticmethod + def process_action(action, event): if action == 'drop': return Procedure.DROP elif action == 'keep': @@ -287,5 +290,4 @@ def get_linecol(model_obj, as_dict=False): return tup - BOT = SieveExpertBot From c8f61348a0e04a42ef1b191906843c75943db09b Mon Sep 17 00:00:00 2001 From: Antoine Neuenschwander Date: Mon, 16 Oct 2017 10:22:28 +0200 Subject: [PATCH 40/40] Validate IP addresses according to harmonization, antoinet/intelmq#11 --- intelmq/bots/experts/sieve/expert.py | 44 ++++++++++++++++--- .../tests/bots/experts/sieve/test_expert.py | 12 +++++ .../test_string_invalid_ipaddr.sieve | 3 ++ 3 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_invalid_ipaddr.sieve diff --git a/intelmq/bots/experts/sieve/expert.py b/intelmq/bots/experts/sieve/expert.py index b8991f2f9..a2135fc38 100644 --- a/intelmq/bots/experts/sieve/expert.py +++ b/intelmq/bots/experts/sieve/expert.py @@ -48,8 +48,14 @@ def init_metamodel(): try: grammarfile = os.path.join(os.path.dirname(__file__), 'sieve.tx') metamodel = metamodel_from_file(grammarfile) - metamodel.register_obj_processors({'SingleIpRange': SieveExpertBot.validate_ip_range}) - metamodel.register_obj_processors({'NumericMatch': SieveExpertBot.validate_numeric_match}) + + # apply custom validation rules + metamodel.register_obj_processors({ + 'StringMatch': SieveExpertBot.validate_string_match, + 'NumericMatch': SieveExpertBot.validate_numeric_match, + 'SingleIpRange': SieveExpertBot.validate_ip_range + }) + return metamodel except TextXError as e: raise ValueError('Could not process sieve grammar file. Error in (%d, %d): %s' % (e.line, e.col, str(e))) @@ -238,10 +244,10 @@ def process_action(action, event): @staticmethod def validate_ip_range(ip_range): - position = SieveExpertBot.get_linecol(ip_range, as_dict=True) try: ipaddress.ip_network(ip_range.value) except ValueError: + position = SieveExpertBot.get_linecol(ip_range, as_dict=True) raise TextXSemanticError('Invalid ip range: %s.' % ip_range.value, **position) @staticmethod @@ -251,8 +257,8 @@ def validate_numeric_match(num_match): Checks if the event key (given on the left hand side of the expression) is of a valid type for a numeric match, according the the IntelMQ harmonization. - Raises: - TextXSemanticError: when the key is of an incompatible type for numeric match experessions. + Raises: + TextXSemanticError: when the key is of an incompatible type for numeric match expressions. """ valid_types = ['Integer', 'Float', 'Accuracy', 'ASN'] position = SieveExpertBot.get_linecol(num_match.value, as_dict=True) @@ -265,6 +271,34 @@ def validate_numeric_match(num_match): except KeyError: raise TextXSemanticError('Invalid key: %s.' % num_match.key, **position) + @staticmethod + def validate_string_match(str_match): + """ Validates a string match expression. + + Checks if the type of the value given on the right hand side of the expression matches the event key in the left + hand side, according to the IntelMQ harmonization. + + Raises: + TextXSemanticError: when the value is of incompatible type with the event key. + """ + + # validate IPAddress + ipaddr_types = [k for k, v in SieveExpertBot.harmonization.items() if v['type'] == 'IPAddress'] + if str_match.key in ipaddr_types: + if str_match.value.__class__.__name__ == 'SingleStringValue': + SieveExpertBot.validate_ip_address(str_match.value) + elif str_match.value.__class__.__name__ == 'StringValueList': + for val in str_match.value.values: + SieveExpertBot.validate_ip_address(val) + + @staticmethod + def validate_ip_address(ipaddr): + try: + ipaddress.ip_address(ipaddr.value) + except ValueError: + position = SieveExpertBot.get_linecol(ipaddr, as_dict=True) + raise TextXSemanticError('Invalid ip address: %s.' % ipaddr.value, **position) + @staticmethod def is_numeric(num): """ Returns True if argument is a number (integer or float). """ diff --git a/intelmq/tests/bots/experts/sieve/test_expert.py b/intelmq/tests/bots/experts/sieve/test_expert.py index 2c596dd04..4109cde9a 100644 --- a/intelmq/tests/bots/experts/sieve/test_expert.py +++ b/intelmq/tests/bots/experts/sieve/test_expert.py @@ -417,6 +417,18 @@ def test_string_inverse_regex_match(self): self.run_bot() self.assertMessageEqual(0, event) + def test_string_invalid_ipaddr(self): + """ Tests validation of harmonization for IP addresses. """ + self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), + 'test_sieve_files/test_string_invalid_ipaddr.sieve') + + event = EXAMPLE_INPUT.copy() + self.input_message = event + with self.assertRaises(ValueError) as context: + self.run_bot() + exception = context.exception + self.assertRegex(str(exception), 'Invalid ip address:') + def test_numeric_equal_match(self): """ Test == numeric match """ self.sysconfig['file'] = os.path.join(os.path.dirname(__file__), diff --git a/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_invalid_ipaddr.sieve b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_invalid_ipaddr.sieve new file mode 100644 index 000000000..0910e810b --- /dev/null +++ b/intelmq/tests/bots/experts/sieve/test_sieve_files/test_string_invalid_ipaddr.sieve @@ -0,0 +1,3 @@ +if source.ip == "#fail" { + add comment = "this should fail" +} \ No newline at end of file