From 559f3dd7078ead13bb68f6cae8bee7912fe2ff1e Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 18:18:14 +0200 Subject: [PATCH 1/8] BUG: lib/pipeline: fix argument uptake the arguments given by pipeline_args were not effective, as the load_configuration methods of the pipeline classes still tried to access the parameters at self. also modify the tests to catch a bug like this in the future fixes certtools/intelmq#1875 --- CHANGELOG.md | 1 + intelmq/lib/pipeline.py | 68 ++++++++++++------------------ intelmq/tests/lib/test_pipeline.py | 28 ++++++------ 3 files changed, 44 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 466c48785..01129c37c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,7 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). - Added `separate-raws-table.sql` (PR#1985 by Sebastian Wagner). ### Known issues +- ParserBot: erroneous raw line recovery in error handling (#1850). 2.3.3 (2021-05-31) diff --git a/intelmq/lib/pipeline.py b/intelmq/lib/pipeline.py index 7b43fec0e..6975bf181 100644 --- a/intelmq/lib/pipeline.py +++ b/intelmq/lib/pipeline.py @@ -4,9 +4,8 @@ # -*- coding: utf-8 -*- import time -import inspect from itertools import chain -from typing import Dict, Optional, Union +from typing import Dict, Optional import ssl import redis @@ -67,8 +66,11 @@ class Pipeline(object): # If the class currently holds a message, restricts the actions _has_message = False - def __init__(self, logger, pipeline_args={}, load_balance=False, is_multithreaded=False): - self.pipeline_args = pipeline_args + def __init__(self, logger, pipeline_args: dict = None, load_balance=False, is_multithreaded=False): + if pipeline_args: + self.pipeline_args = pipeline_args + else: + self.pipeline_args = {} self.destination_queues = {} # type: dict[str, list] self.internal_queue = None self.source_queue = None @@ -185,22 +187,16 @@ class Redis(Pipeline): destination_pipeline_password = None def load_configurations(self, queues_type): - self.host = getattr(self.pipeline_args, - "{}_pipeline_host".format(queues_type), - "127.0.0.1") - self.port = getattr(self.pipeline_args, - "{}_pipeline_port".format(queues_type), "6379") - self.db = getattr(self.pipeline_args, - "{}_pipeline_db".format(queues_type), 2) - self.password = getattr(self.pipeline_args, - "{}_pipeline_password".format(queues_type), - None) + self.host = self.pipeline_args.get("{}_pipeline_host".format(queues_type), + "127.0.0.1") + self.port = self.pipeline_args.get("{}_pipeline_port".format(queues_type), "6379") + self.db = self.pipeline_args.get("{}_pipeline_db".format(queues_type), 2) + self.password = self.pipeline_args.get("{}_pipeline_password".format(queues_type), + None) # socket_timeout is None by default, which means no timeout - self.socket_timeout = getattr(self.pipeline_args, - "{}_pipeline_socket_timeout".format( - queues_type), - None) - self.load_balance = getattr(self, "load_balance", False) + self.socket_timeout = self.pipeline_args.get("{}_pipeline_socket_timeout".format(queues_type), + None) + self.load_balance = self.pipeline_args.get("load_balance", False) self.load_balance_iterator = 0 def connect(self): @@ -433,35 +429,25 @@ class Amqp(Pipeline): destination_pipeline_amqp_exchange = "" intelmqctl_rabbitmq_monitoring_url = None - def __init__(self, logger, pipeline_args={}, load_balance=False, is_multithreaded=False): + def __init__(self, logger, pipeline_args: dict = None, load_balance=False, is_multithreaded=False): super(Amqp, self).__init__(logger, pipeline_args, load_balance, is_multithreaded) if pika is None: raise ValueError("To use AMQP you must install the 'pika' library.") self.properties = pika.BasicProperties(delivery_mode=2) # message persistence def load_configurations(self, queues_type): - self.host = getattr(self, - "{}_pipeline_host".format(queues_type), - "127.0.0.1") - self.port = getattr(self, - "{}_pipeline_port".format(queues_type), 5672) - self.username = getattr(self, - "{}_pipeline_username".format(queues_type), - None) - self.password = getattr(self, - "{}_pipeline_password".format(queues_type), - None) + self.host = self.pipeline_args.get("{}_pipeline_host".format(queues_type), "10.0.0.1") + self.port = self.pipeline_args.get("{}_pipeline_port".format(queues_type), 5672) + self.username = self.pipeline_args.get("{}_pipeline_username".format(queues_type), None) + self.password = self.pipeline_args.get("{}_pipeline_password".format(queues_type), None) # socket_timeout is None by default, which means no timeout - self.socket_timeout = getattr(self, - "{}_pipeline_socket_timeout".format( - queues_type), - None) - self.load_balance = getattr(self, "load_balance", False) - self.virtual_host = getattr(self, - "{}_pipeline_amqp_virtual_host".format(queues_type), - '/') - self.ssl = getattr(self, "{}_pipeline_ssl".format(queues_type), False) - self.exchange = getattr(self, "{}_pipeline_amqp_exchange".format(queues_type), "") + self.socket_timeout = self.pipeline_args.get("{}_pipeline_socket_timeout".format(queues_type), + None) + self.load_balance = self.pipeline_args.get("load_balance", False) + self.virtual_host = self.pipeline_args.get("{}_pipeline_amqp_virtual_host".format(queues_type), + '/') + self.ssl = self.pipeline_args.get("{}_pipeline_ssl".format(queues_type), False) + self.exchange = self.pipeline_args.get("{}_pipeline_amqp_exchange".format(queues_type), "") self.load_balance_iterator = 0 self.kwargs = {} if self.username and self.password: diff --git a/intelmq/tests/lib/test_pipeline.py b/intelmq/tests/lib/test_pipeline.py index c32b681ca..39f75eb0f 100644 --- a/intelmq/tests/lib/test_pipeline.py +++ b/intelmq/tests/lib/test_pipeline.py @@ -29,9 +29,6 @@ } -class Parameters(object): - pass - class TestPipeline(unittest.TestCase): @@ -140,16 +137,18 @@ class TestRedis(unittest.TestCase): """ def setUp(self): - params = Parameters() - setattr(params, 'source_pipeline_host', os.getenv('INTELMQ_PIPELINE_HOST', 'localhost')) - setattr(params, 'source_pipeline_password', os.getenv('INTELMQ_TEST_REDIS_PASSWORD')) - setattr(params, 'source_pipeline_db', 4) - setattr(params, 'destination_pipeline_host', os.getenv('INTELMQ_PIPELINE_HOST', 'localhost')) - setattr(params, 'destination_pipeline_password', os.getenv('INTELMQ_TEST_REDIS_PASSWORD')) - setattr(params, 'destination_pipeline_db', 4) + params = {} + params['source_pipeline_host'] = os.getenv('INTELMQ_PIPELINE_HOST', 'localhost') + params['source_pipeline_password'] = os.getenv('INTELMQ_TEST_REDIS_PASSWORD') + params['source_pipeline_db'] = 4 + params['destination_pipeline_host'] = os.getenv('INTELMQ_PIPELINE_HOST', 'localhost') + params['destination_pipeline_password'] = os.getenv('INTELMQ_TEST_REDIS_PASSWORD') + params['destination_pipeline_db'] = 4 logger = logging.getLogger('foo') logger.addHandler(logging.NullHandler()) - self.pipe = pipeline.PipelineFactory.create(logger, broker='Redis', pipeline_args=params.__dict__) + self.pipe = pipeline.PipelineFactory.create(logger, broker='Redis', pipeline_args=params) + self.pipe.source_pipeline_host = '10.0.0.1' # force fail if load_configuration is ineffective (#1875) + self.pipe.load_configurations('source') self.pipe.set_queues('test', 'source') self.pipe.set_queues('test', 'destination') self.pipe.connect() @@ -216,7 +215,12 @@ class TestAmqp(unittest.TestCase): def setUp(self): logger = logging.getLogger('foo') logger.addHandler(logging.NullHandler()) - self.pipe = pipeline.PipelineFactory.create(logger=logger, broker='Amqp') + + self.pipe = pipeline.PipelineFactory.create(logger=logger, broker='Amqp', + pipeline_args={'source_pipeline_host': '127.0.0.1', + 'destination_pipeline_host': '127.0.0.1'}) + self.pipe.source_pipeline_host = '10.0.0.1' # force fail if load_configuration is ineffective (#1875) + self.pipe.load_configurations('source') self.pipe.set_queues('test', 'source') self.pipe.set_queues('test', 'destination') self.pipe.connect() From 97c07f47f70886468628c27d93e91d0182ccad0a Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 18:43:33 +0200 Subject: [PATCH 2/8] DEP: rmeove deprecated ripencc expert was renamed and marked as deprecated in 2.0.0.beta1 certtools/intelmq#1404 --- CHANGELOG.md | 1 + NEWS.md | 4 ++++ .../experts/ripencc_abuse_contact/__init__.py | 0 .../experts/ripencc_abuse_contact/expert.py | 19 ------------------- .../tests/bots/experts/ripe/test_expert.py | 2 +- .../experts/ripencc_abuse_contact/__init__.py | 0 6 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 intelmq/bots/experts/ripencc_abuse_contact/__init__.py delete mode 100644 intelmq/bots/experts/ripencc_abuse_contact/expert.py delete mode 100644 intelmq/tests/bots/experts/ripencc_abuse_contact/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 01129c37c..09efece1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). - Split string and numeric matches into single- and multivalued variants, with the relevant new operators `:in`, `:containsany` and `:regexin` for string lists, and `:in` for numeric value lists (PR#1957 by Mikk Margus Möll). - Removed the `==` operator for lists, with the previous meaning of `:in`. Have a look at the NEWS.md for more information. - Added `intelmq.bots.experts.uwhoisd`: A bot that fetches the whois entry from a uwhois-instance (PR#1918 by Raphaël Vinot). +- Removed deprecated `intelmq.bots.experts.ripencc_abuse_contact.expert`. It was replaced by `intelmq.bots.experts.ripe.expert` and marked as deprecated in 2.0.0.beta1 (PR#1997 by Sebastian Wagner, #1404). #### Outputs - Remove `intelmq.bots.outputs.xmpp`: one of the dependencies of the bot was deprecated and according to a short survey on the IntelMQ diff --git a/NEWS.md b/NEWS.md index 7aedce537..22aa9c309 100644 --- a/NEWS.md +++ b/NEWS.md @@ -85,6 +85,10 @@ The `pipeline.conf` file was removed. The source- and destination-queues of the The `intelmqctl upgrade-config` command migrates the existing configuration from the `pipeline.conf` file to the individual bot configurations in the `runtime.conf` configuration file. The `runtime.conf` file was replaced by a `runtime.yaml` file. IntelMQ moves the file for you if it does not find a runtime.conf but a runtime.yaml file. When IntelMQ changes the file, it now writes YAML syntax. +#### Removal of deprecated bots and behaviour +- The bot `intelmq.bots.experts.ripencc_abuse_contact.expert` has been removed. It was replaced by `intelmq.bots.experts.ripe.expert` and marked as deprecated in 2.0.0.beta1. + + ### Libraries ### Postgres databases diff --git a/intelmq/bots/experts/ripencc_abuse_contact/__init__.py b/intelmq/bots/experts/ripencc_abuse_contact/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/intelmq/bots/experts/ripencc_abuse_contact/expert.py b/intelmq/bots/experts/ripencc_abuse_contact/expert.py deleted file mode 100644 index c59330e1d..000000000 --- a/intelmq/bots/experts/ripencc_abuse_contact/expert.py +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: 2015 National CyberSecurity Center -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -from ..ripe.expert import RIPEExpertBot - - -class RIPENCCExpertDeprecatedBot(RIPEExpertBot): - - def init(self): - self.logger.warning("The parser 'intelmq.bots.experts.ripencc_abuse_contact" - ".expert has been renamed to 'intelmq.bots." - "experts.ripe.expert'. This compatibility module " - "will be removed in version 3.0.") - super().init() - - -BOT = RIPENCCExpertDeprecatedBot diff --git a/intelmq/tests/bots/experts/ripe/test_expert.py b/intelmq/tests/bots/experts/ripe/test_expert.py index d50a8b686..cf8717994 100644 --- a/intelmq/tests/bots/experts/ripe/test_expert.py +++ b/intelmq/tests/bots/experts/ripe/test_expert.py @@ -12,7 +12,7 @@ import requests_mock import intelmq.lib.test as test -from intelmq.bots.experts.ripencc_abuse_contact.expert import RIPEExpertBot +from intelmq.bots.experts.ripe.expert import RIPEExpertBot EXAMPLE_INPUT = {"__type": "Event", "source.ip": "93.184.216.34", # example.com diff --git a/intelmq/tests/bots/experts/ripencc_abuse_contact/__init__.py b/intelmq/tests/bots/experts/ripencc_abuse_contact/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 514d6120439d07745ca5ff19efd43916ede8e46a Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 18:55:00 +0200 Subject: [PATCH 3/8] DEP: intelmqdump: rename commands e and v The command `e` for deleting single entries by given IDs has been merged into the command `d` ("delete"), which can now delete either entries by ID or the whole file. The command `v` for editing entries has been renamed to `e` ("edit"). --- CHANGELOG.md | 3 ++ NEWS.md | 4 +++ docs/user/configuration-management.rst | 16 +++++----- intelmq/bin/intelmqdump.py | 43 +++++++++++++------------- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09efece1d..c5519de1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,6 +130,9 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). - Compare content of the `output` field as dictionaries, not as string in `assertMessageEqual` (PR#1975 by Karl-Johan Karlsson). ### Tools +- intelmqdump (PR#1997 by Sebastian Wagner, #1404): + - The command `e` for deleting single entries by given IDs has been merged into the command `d` ("delete"), which can now delete either entries by ID or the whole file. + - The command `v` for editing entries has been renamed to `e` ("edit"). ### Contrib - eventdb: diff --git a/NEWS.md b/NEWS.md index 22aa9c309..efe8e76b7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -17,6 +17,10 @@ IntelMQ now uses YAML for the runtime configuration and therefore needs the `rua ### Tools +#### intelmqdump +The command `e` for deleting single entries by given IDs has been merged into the command `d` ("delete"), which can now delete either entries by ID or the whole file. +The command `v` for editing entries has been renamed to `e` ("edit"). + ### Bots Both the XMPP collector bot and the XMPP output bot were removed. This [was evaluated on the mailinglist](https://lists.cert.at/pipermail/intelmq-users/2020-October/000177.html) diff --git a/docs/user/configuration-management.rst b/docs/user/configuration-management.rst index 060056825..1c16a5f23 100644 --- a/docs/user/configuration-management.rst +++ b/docs/user/configuration-management.rst @@ -582,9 +582,9 @@ When bots are failing due to bad input data or programming errors, they can dump > a modify-expert-queue All messages in the opened file will be recovered to the stored or given queue and removed from the file. - - e, Delete entries by IDs - > e id{,id} - > e 3,5 + - d, Delete entries by IDs + > d id{,id} + > d 3,5 The entries will be deleted from the dump file. - d, Delete file > d @@ -594,10 +594,10 @@ When bots are failing due to bad input data or programming errors, they can dump > s 0,4,5 Show the selected IP in a readable format. It's still a raw format from repr, but with newlines for message and traceback. - - v, Edit by ID - > v id - > v 0 - > v 1,2 + - e, Edit by ID + > e id + > e 0 + > e 1,2 Opens an editor (by calling `sensible-editor`) on the message. The modified message is then saved in the dump. - q, Quit > q @@ -616,7 +616,7 @@ When bots are failing due to bad input data or programming errors, they can dump Processing dragon-research-group-ssh-parser: 2 dumps 0: 2015-09-03T13:13:22.159014 InvalidValue: invalid value u'NA' () for key u'source.asn' 1: 2015-09-01T14:40:20.973743 InvalidValue: invalid value u'NA' () for key u'source.asn' - recover (a)ll, delete (e)ntries, (d)elete file, (q)uit, (s)how by ids, (r)ecover by ids? d + (r)ecover by ids, recover (a)ll, delete (e)ntries, (d)elete file, (s)how by ids, (q)uit, edit id (v)? d Deleted file /opt/intelmq/var/log/dragon-research-group-ssh-parser.dump Bots and the intelmqdump tool use file locks to prevent writing to already opened files. Bots are trying to lock the file for up to 60 seconds if the dump file is locked already by another process (intelmqdump) and then give up. Intelmqdump does not wait and instead only shows an error message. diff --git a/intelmq/bin/intelmqdump.py b/intelmq/bin/intelmqdump.py index bb5177e25..9f1d5b87f 100644 --- a/intelmq/bin/intelmqdump.py +++ b/intelmq/bin/intelmqdump.py @@ -51,9 +51,9 @@ > a modify-expert-queue All messages in the opened file will be recovered to the stored or given queue and removed from the file. -- e, Delete entries by IDs - > e id{,id} - > e 3,5 +- d, Delete entries by IDs + > d id{,id} + > d 3,5 The entries will be deleted from the dump file. - d, Delete file > d @@ -63,10 +63,10 @@ > s 0,4,5 Show the selected IP in a readable format. It's still a raw format from repr, but with newlines for message and traceback. -- v, Edit by ID - > v id - > v 0 - > v 1,2 +- e, Edit by ID + > e id + > e 0 + > e 1,2 Opens an editor (by calling `sensible-editor`) on the message. The modified message is then saved in the dump. - q, Quit > q @@ -77,11 +77,10 @@ # shortcut: description, takes ids, available for corrupted files ACTIONS = {'r': ('(r)ecover by ids', True, False), 'a': ('recover (a)ll', False, False), - 'e': ('delete (e)ntries', True, False), - 'd': ('(d)elete file', False, True), + 'd': ('(d)elete file or entries by id', True, False), 's': ('(s)how by ids', True, False), 'q': ('(q)uit', False, True), - 'v': ('edit id (v)', True, False), + 'e': ('(e)dit by id', True, False), } AVAILABLE_IDS = [key for key, value in ACTIONS.items() if value[1]] @@ -273,7 +272,7 @@ def main(): print('Restricted actions.') else: # don't display list after 'show', 'recover' & edit commands - if not (answer and isinstance(answer, list) and answer[0] in ['s', 'r', 'v']): + if not (answer and isinstance(answer, list) and answer[0] in ['s', 'r', 'e']): content = json.load(handle) handle.seek(0) content = OrderedDict(sorted(content.items(), key=lambda t: t[0])) # sort by key here, #1280 @@ -325,11 +324,6 @@ def main(): queue_name = answer[1] if answer[0] == 'q': break - elif answer[0] == 'e': - # Delete entries - for entry in ids: - del content[meta[entry][0]] - save_file(handle, content) elif answer[0] == 'r': # recover entries params = defaults.copy() @@ -376,10 +370,17 @@ def main(): print('Deleting empty file {}'.format(fname)) break elif answer[0] == 'd': - # delete dumpfile - delete_file = True - print('Deleting empty file {}'.format(fname)) - break + # Delete entries or file + if ids: + # delete entries + for entry in ids: + del content[meta[entry][0]] + save_file(handle, content) + else: + # delete dumpfile + delete_file = True + print('Deleting file {}'.format(fname)) + break elif answer[0] == 's': # Show entries by id for count, (key, orig_value) in enumerate(content.items()): @@ -401,7 +402,7 @@ def main(): if type(value['traceback']) is not list: value['traceback'] = value['traceback'].splitlines() pprint.pprint(value) - elif answer[0] == 'v': + elif answer[0] == 'e': # edit given id if not ids: print(red('Edit mode needs an id')) From 7fc8bd48fd0e9aa596cf1712787bb5e422ef016e Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 19:00:08 +0200 Subject: [PATCH 4/8] DEP: modify expert: remove compat with old format Compatibility with the deprecated configuration format (before 1.0.0.dev7) was removed. certtools/intelmq#1404 --- CHANGELOG.md | 2 ++ NEWS.md | 1 + intelmq/bots/experts/modify/expert.py | 6 ------ intelmq/tests/bots/experts/modify/old_format.conf | 9 --------- .../bots/experts/modify/old_format.conf.license | 2 -- intelmq/tests/bots/experts/modify/test_expert.py | 13 +------------ 6 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 intelmq/tests/bots/experts/modify/old_format.conf delete mode 100644 intelmq/tests/bots/experts/modify/old_format.conf.license diff --git a/CHANGELOG.md b/CHANGELOG.md index c5519de1d..ed44cbb65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,8 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). - Removed the `==` operator for lists, with the previous meaning of `:in`. Have a look at the NEWS.md for more information. - Added `intelmq.bots.experts.uwhoisd`: A bot that fetches the whois entry from a uwhois-instance (PR#1918 by Raphaël Vinot). - Removed deprecated `intelmq.bots.experts.ripencc_abuse_contact.expert`. It was replaced by `intelmq.bots.experts.ripe.expert` and marked as deprecated in 2.0.0.beta1 (PR#1997 by Sebastian Wagner, #1404). +- `intelmq.bots.experts.modify.expert`: + - Removed compatibility with deprecated configuration format before 1.0.0.dev7 (PR#1997 by Sebastian Wagner, #1404). #### Outputs - Remove `intelmq.bots.outputs.xmpp`: one of the dependencies of the bot was deprecated and according to a short survey on the IntelMQ diff --git a/NEWS.md b/NEWS.md index efe8e76b7..4435a594b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -91,6 +91,7 @@ The `runtime.conf` file was replaced by a `runtime.yaml` file. IntelMQ moves the #### Removal of deprecated bots and behaviour - The bot `intelmq.bots.experts.ripencc_abuse_contact.expert` has been removed. It was replaced by `intelmq.bots.experts.ripe.expert` and marked as deprecated in 2.0.0.beta1. +- Modify expert: Compatibility with the deprecated configuration format (before 1.0.0.dev7) was removed. ### Libraries diff --git a/intelmq/bots/experts/modify/expert.py b/intelmq/bots/experts/modify/expert.py index 146ba0e2d..ac32d14eb 100644 --- a/intelmq/bots/experts/modify/expert.py +++ b/intelmq/bots/experts/modify/expert.py @@ -11,7 +11,6 @@ from intelmq.lib.bot import Bot from intelmq.lib.utils import load_configuration -from intelmq.lib.upgrades import modify_expert_convert_config def is_re_pattern(value): @@ -47,11 +46,6 @@ class ModifyExpertBot(Bot): def init(self): config = load_configuration(self.configuration_path) - if type(config) is dict: - self.logger.warning('Support for dict-based configuration will be ' - 'removed in version 3.0. Have a look at the ' - 'NEWS file section 1.0.0.dev7.') - config = modify_expert_convert_config(config) if self.case_sensitive: self.re_kwargs = {} diff --git a/intelmq/tests/bots/experts/modify/old_format.conf b/intelmq/tests/bots/experts/modify/old_format.conf deleted file mode 100644 index 0cb540455..000000000 --- a/intelmq/tests/bots/experts/modify/old_format.conf +++ /dev/null @@ -1,9 +0,0 @@ -{ -"Blocklist.de": { - "__default": [{ - "feed.name": "^BlockList\\.de$", - "classification.identifier": "" - }, { - }] - } -} diff --git a/intelmq/tests/bots/experts/modify/old_format.conf.license b/intelmq/tests/bots/experts/modify/old_format.conf.license deleted file mode 100644 index f0b62ad2d..000000000 --- a/intelmq/tests/bots/experts/modify/old_format.conf.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2016 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/experts/modify/test_expert.py b/intelmq/tests/bots/experts/modify/test_expert.py index 3ec38ecda..5c9111f82 100644 --- a/intelmq/tests/bots/experts/modify/test_expert.py +++ b/intelmq/tests/bots/experts/modify/test_expert.py @@ -13,7 +13,7 @@ import intelmq.lib.test as test from intelmq.lib.utils import load_configuration -from intelmq.bots.experts.modify.expert import ModifyExpertBot, modify_expert_convert_config +from intelmq.bots.experts.modify.expert import ModifyExpertBot EVENT_TEMPL = {"__type": "Event", "feed.name": "Spamhaus Cert", @@ -92,17 +92,6 @@ def test_events(self): for position, event_out in enumerate(OUTPUT[:7]): self.assertMessageEqual(position, event_out) - def test_conversion(self): - """ Test if the conversion from old dict-based config to new list based is correct. """ - old_path = resource_filename('intelmq', - 'tests/bots/experts/modify/old_format.conf') - old_config = load_configuration(old_path) - new_path = resource_filename('intelmq', - 'tests/bots/experts/modify/new_format.conf') - new_config = load_configuration(new_path) - self.assertDictEqual(modify_expert_convert_config(old_config)[0], - new_config[0]) - def test_types(self): """ boolean, int etc From 44b6b6b3913339d3af1dc770243157b407be597f Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 19:05:50 +0200 Subject: [PATCH 5/8] DEP: rt collector: removed parameter unzip_attachment compatibility with the deprecated parameter `unzip_attachment` (removed in 2.1.0) was removed. certtools/intelmq#1404 --- CHANGELOG.md | 2 ++ NEWS.md | 1 + intelmq/bots/collectors/rt/collector_rt.py | 12 ------------ intelmq/tests/bots/collectors/rt/test_collector.py | 4 ---- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed44cbb65..95b745352 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,8 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). - `intelmq.bots.collectors.api.collector_api` (PR#1987 by Mikk Margus Möll, fixes #1986): - Added UNIX socket capability. - Correctly close the IOLoop in the shutdown method to fix reload. +- `intelmq.bots.collectors.rt.collector_rt` (PR#1997 by Sebastian Wagner, #1404): + - compatibility with the deprecated parameter `unzip_attachment` (removed in 2.1.0) was removed. #### Parsers - Added `intelmq.bots.parsers.fireeye`: A bot that parses hashes and URLs from Fireeye MAS indicators (PR#1745 by Christopher Schappelwein). diff --git a/NEWS.md b/NEWS.md index 4435a594b..8bf02b27d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -92,6 +92,7 @@ The `runtime.conf` file was replaced by a `runtime.yaml` file. IntelMQ moves the #### Removal of deprecated bots and behaviour - The bot `intelmq.bots.experts.ripencc_abuse_contact.expert` has been removed. It was replaced by `intelmq.bots.experts.ripe.expert` and marked as deprecated in 2.0.0.beta1. - Modify expert: Compatibility with the deprecated configuration format (before 1.0.0.dev7) was removed. +- RT collector: compatibility with the deprecated parameter `unzip_attachment` (removed in 2.1.0) was removed. ### Libraries diff --git a/intelmq/bots/collectors/rt/collector_rt.py b/intelmq/bots/collectors/rt/collector_rt.py index d9e56377b..c066511e6 100644 --- a/intelmq/bots/collectors/rt/collector_rt.py +++ b/intelmq/bots/collectors/rt/collector_rt.py @@ -71,18 +71,6 @@ def init(self): self._parse_extract_file_parameter('extract_attachment') self._parse_extract_file_parameter('extract_download') - if hasattr(self, 'unzip_attachment'): - self.logger.warning("The parameter 'unzip_attachment' is deprecated and " - "will be removed in version 3.0 in favor of the " - "more generic and powerful 'extract_attachment'. " - "Look at the Bots documentation for more details.") - if not self.extract_attachment: - self.extract_attachment = self.unzip_attachment - else: - self.logger.warn("Both 'extract_attachment' and the deprecated " - "'unzip_attachment' parameter are in use. Ignoring " - "the latter one.") - def process(self): RT = rt.Rt(self.uri, self.user, self.password) diff --git a/intelmq/tests/bots/collectors/rt/test_collector.py b/intelmq/tests/bots/collectors/rt/test_collector.py index 9ab9af9b1..1a8a242b9 100644 --- a/intelmq/tests/bots/collectors/rt/test_collector.py +++ b/intelmq/tests/bots/collectors/rt/test_collector.py @@ -62,7 +62,6 @@ def set_bot(cls): 'search_status': 'new', 'attachment_regex': '.*.zip', 'url_regex': None, - 'unzip_attachment': True, 'name': 'Example feed', 'extract_attachment': True, } @@ -73,7 +72,6 @@ def test_attachment_zip(self): """ Test a zipped attachment """ - self.allowed_warning_count = 2 self.run_bot(iterations=1) self.assertMessageEqual(0, REPORT) @@ -83,7 +81,6 @@ def test_attachment_gz(self): """ Test a gzipped attachment """ - self.allowed_warning_count = 2 self.prepare_bot(parameters={'attachment_regex': r'.*\.gz'}) self.run_bot(iterations=1, prepare=False) self.assertMessageEqual(0, REPORT) @@ -94,7 +91,6 @@ def test_url_zip(self): """ Test a zipped URL """ - self.allowed_warning_count = 2 self.prepare_bot(parameters={'attachment_regex': None, 'url_regex': r'http://localhost/.*\.zip', 'extract_download': True}) From 8eed6fdbe9fea67086610106019145de936f7375 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 19:08:49 +0200 Subject: [PATCH 6/8] MAINT: reverse dns expert: remove obsolete code the init contained a fallback method if a parameter does not exist this is no longer necessary, as all bots have proper default values related to certtools/intelmq#1404 --- intelmq/bots/experts/reverse_dns/expert.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/intelmq/bots/experts/reverse_dns/expert.py b/intelmq/bots/experts/reverse_dns/expert.py index 075470a7f..309befb71 100644 --- a/intelmq/bots/experts/reverse_dns/expert.py +++ b/intelmq/bots/experts/reverse_dns/expert.py @@ -33,12 +33,6 @@ class ReverseDnsExpertBot(Bot, CacheMixin): redis_cache_port: int = 6379 redis_cache_ttl: int = 86400 - def init(self): - if not hasattr(self, 'overwrite'): - self.logger.warning("Parameter 'overwrite' is not given, assuming 'True'. " - "Please set it explicitly, default will change to " - "'False' in version 3.0.0'.") - def process(self): event = self.receive_message() From 6143f9c61a9a1137cb3266178db60487fd3eaf57 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 19:13:42 +0200 Subject: [PATCH 7/8] DEP: lib/utils: remove deprecated function create_request_session_from_bot was deprecated and is no longer used certtools/intelmq#1404 --- CHANGELOG.md | 1 + intelmq/lib/utils.py | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95b745352..b747d27d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ CHANGELOG - `intelmq.lib.utils`: - New function `list_all_bots` to list all available/installed bots as replacement for the BOTS file (#368, #552, #644, #757, #1069, #1750, PR#1751 by Sebastian Waldbauer). - New function `get_bots_settings` to return the effective bot parameters, with global parameters applied. + - Removed deprecated function `create_request_session_from_bot` (PR#1997 by Sebastian Wagner, #1404). - `intelmq.lib.bot_debugger`: - Set bot's `logging_level` directly in `__init__` before the bot's initialization by changing the default value (by Sebastian Wagner). - Rewrite `load_configuration_patch` by adapting it to the parameter and configuration rewrite (by Sebastian Wagner). diff --git a/intelmq/lib/utils.py b/intelmq/lib/utils.py index 804cc1934..6dee57db0 100644 --- a/intelmq/lib/utils.py +++ b/intelmq/lib/utils.py @@ -744,13 +744,6 @@ def send(self, *args, **kwargs): return super().send(*args, **kwargs) -def create_request_session_from_bot(bot: type) -> requests.Session: - warnings.warn("This function is deprecated in favor of create_request_session" - " and will be removed in version 3.0.0.", - DeprecationWarning) - return create_request_session(bot) - - def create_request_session(bot: type = None) -> requests.Session: """ Creates a requests.Session object preconfigured with the parameters From 63a9e461f6b9f0b867eb0388fd0995d277f60378 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 17 Jun 2021 19:20:20 +0200 Subject: [PATCH 8/8] DEP: remove deprecated database update scripts The deprecated shell scripts - `update-asn-data` - `update-geoip-data` - `update-tor-nodes` - `update-rfiprisk-data` have been removed in favor of the built-in update-mechanisms (see the bots' documentation). A crontab file for calling all new update command can be found in `contrib/cron-jobs/intelmq-update-database`. certtools/intelmq#1404 --- CHANGELOG.md | 6 ++++++ NEWS.md | 8 ++++++++ contrib/cron-jobs/update-asn-data | 1 - contrib/cron-jobs/update-geoip-data | 1 - contrib/cron-jobs/update-tor-nodes | 1 - debian/rules | 9 --------- intelmq/bots/experts/asn_lookup/update-asn-data | 6 ------ intelmq/bots/experts/asn_lookup/update-asn-data.license | 2 -- intelmq/bots/experts/maxmind_geoip/update-geoip-data | 6 ------ .../bots/experts/maxmind_geoip/update-geoip-data.license | 2 -- .../experts/recordedfuture_iprisk/update-rfiprisk-data | 6 ------ .../recordedfuture_iprisk/update-rfiprisk-data.license | 2 -- intelmq/bots/experts/tor_nodes/update-tor-nodes | 6 ------ intelmq/bots/experts/tor_nodes/update-tor-nodes.license | 2 -- setup.py | 4 ---- 15 files changed, 14 insertions(+), 48 deletions(-) delete mode 120000 contrib/cron-jobs/update-asn-data delete mode 120000 contrib/cron-jobs/update-geoip-data delete mode 120000 contrib/cron-jobs/update-tor-nodes delete mode 100755 intelmq/bots/experts/asn_lookup/update-asn-data delete mode 100644 intelmq/bots/experts/asn_lookup/update-asn-data.license delete mode 100755 intelmq/bots/experts/maxmind_geoip/update-geoip-data delete mode 100644 intelmq/bots/experts/maxmind_geoip/update-geoip-data.license delete mode 100755 intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data delete mode 100644 intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data.license delete mode 100755 intelmq/bots/experts/tor_nodes/update-tor-nodes delete mode 100644 intelmq/bots/experts/tor_nodes/update-tor-nodes.license diff --git a/CHANGELOG.md b/CHANGELOG.md index b747d27d1..7062f99bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,12 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). ### Contrib - eventdb: - Added `separate-raws-table.sql` (PR#1985 by Sebastian Wagner). +- cron-jobs: Removed the deprecated update scripts (PR#1997 by Sebastian Wagner, #1404): + - `update-asn-data` + - `update-geoip-data` + - `update-tor-nodes` + - `update-rfiprisk-data` + in favor of the built-in update-mechanisms (see the bots' documentation). A crontab file for calling all new update command can be found in `contrib/cron-jobs/intelmq-update-database`. ### Known issues - ParserBot: erroneous raw line recovery in error handling (#1850). diff --git a/NEWS.md b/NEWS.md index 8bf02b27d..e031914e9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -21,6 +21,14 @@ IntelMQ now uses YAML for the runtime configuration and therefore needs the `rua The command `e` for deleting single entries by given IDs has been merged into the command `d` ("delete"), which can now delete either entries by ID or the whole file. The command `v` for editing entries has been renamed to `e` ("edit"). +#### Cronjobs +The deprecated shell scripts +- `update-asn-data` +- `update-geoip-data` +- `update-tor-nodes` +- `update-rfiprisk-data` +have been removed in favor of the built-in update-mechanisms (see the bots' documentation). A crontab file for calling all new update command can be found in `contrib/cron-jobs/intelmq-update-database`. + ### Bots Both the XMPP collector bot and the XMPP output bot were removed. This [was evaluated on the mailinglist](https://lists.cert.at/pipermail/intelmq-users/2020-October/000177.html) diff --git a/contrib/cron-jobs/update-asn-data b/contrib/cron-jobs/update-asn-data deleted file mode 120000 index f863b57cf..000000000 --- a/contrib/cron-jobs/update-asn-data +++ /dev/null @@ -1 +0,0 @@ -../../intelmq/bots/experts/asn_lookup/update-asn-data \ No newline at end of file diff --git a/contrib/cron-jobs/update-geoip-data b/contrib/cron-jobs/update-geoip-data deleted file mode 120000 index c052a97b7..000000000 --- a/contrib/cron-jobs/update-geoip-data +++ /dev/null @@ -1 +0,0 @@ -../../intelmq/bots/experts/maxmind_geoip/update-geoip-data \ No newline at end of file diff --git a/contrib/cron-jobs/update-tor-nodes b/contrib/cron-jobs/update-tor-nodes deleted file mode 120000 index 956ab9347..000000000 --- a/contrib/cron-jobs/update-tor-nodes +++ /dev/null @@ -1 +0,0 @@ -../../intelmq/bots/experts/tor_nodes/update-tor-nodes \ No newline at end of file diff --git a/debian/rules b/debian/rules index ddf33120f..3d8197361 100755 --- a/debian/rules +++ b/debian/rules @@ -37,15 +37,6 @@ override_dh_auto_install: $(BOTDOCS) sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py sed -i -f debian/sedfile intelmq/etc/* docs/user/intelmqctl.rst docs/user/bots.rst setup.py contrib/logrotate/intelmq contrib/logcheck/logcheck.logfiles python3 setup.py install --root=debian/intelmq --prefix=/usr - # these are already in /usr/bin/ - #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/maxmind_geoip/update-geoip-data - #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/asn_lookup/update-asn-data - #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/tor_nodes/update-tor-nodes - # and rename those in /usr/bin - mv debian/intelmq/usr/bin/update-geoip-data debian/intelmq/usr/bin/intelmq-update-geoip-data - mv debian/intelmq/usr/bin/update-asn-data debian/intelmq/usr/bin/intelmq-update-asn-data - mv debian/intelmq/usr/bin/update-tor-nodes debian/intelmq/usr/bin/intelmq-update-tor-nodes - mv debian/intelmq/usr/bin/update-rfiprisk-data debian/intelmq/usr/bin/intelmq-update-rfiprisk-data # create directories mkdir -p debian/intelmq/var/log/intelmq mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output diff --git a/intelmq/bots/experts/asn_lookup/update-asn-data b/intelmq/bots/experts/asn_lookup/update-asn-data deleted file mode 100755 index 7e0a624f1..000000000 --- a/intelmq/bots/experts/asn_lookup/update-asn-data +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -echo 'This script is deprecated and will be removed in version 3.0. -Please use the command "intelmq.bots.experts.asn_lookup.expert --update-database" instead.' > /dev/stderr - -intelmq.bots.experts.asn_lookup.expert --update-database diff --git a/intelmq/bots/experts/asn_lookup/update-asn-data.license b/intelmq/bots/experts/asn_lookup/update-asn-data.license deleted file mode 100644 index 8fd09f5e0..000000000 --- a/intelmq/bots/experts/asn_lookup/update-asn-data.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2016 Sascha Wilde -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/bots/experts/maxmind_geoip/update-geoip-data b/intelmq/bots/experts/maxmind_geoip/update-geoip-data deleted file mode 100755 index dbba9305a..000000000 --- a/intelmq/bots/experts/maxmind_geoip/update-geoip-data +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -echo 'This script is deprecated and will be removed in version 3.0. -Please use the command "intelmq.bots.experts.maxmind_geoip.expert --update-database" instead.' > /dev/stderr - -intelmq.bots.experts.maxmind_geoip.expert --update-database diff --git a/intelmq/bots/experts/maxmind_geoip/update-geoip-data.license b/intelmq/bots/experts/maxmind_geoip/update-geoip-data.license deleted file mode 100644 index 8fd09f5e0..000000000 --- a/intelmq/bots/experts/maxmind_geoip/update-geoip-data.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2016 Sascha Wilde -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data b/intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data deleted file mode 100755 index b277aedfb..000000000 --- a/intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -echo 'This script is deprecated and will be removed in version 3.0. -Please use the command "intelmq.bots.experts.recordedfuture_iprisk.expert --update-database" instead.' > /dev/stderr - -intelmq.bots.experts.recordedfuture_iprisk.expert --update-database diff --git a/intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data.license b/intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data.license deleted file mode 100644 index 42e75399f..000000000 --- a/intelmq/bots/experts/recordedfuture_iprisk/update-rfiprisk-data.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2018 olekristoffer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/bots/experts/tor_nodes/update-tor-nodes b/intelmq/bots/experts/tor_nodes/update-tor-nodes deleted file mode 100755 index fa6de58ef..000000000 --- a/intelmq/bots/experts/tor_nodes/update-tor-nodes +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -echo 'This script is deprecated and will be removed in version 3.0. -Please use the command "intelmq.bots.experts.tor_nodes.expert --update-database" instead.' > /dev/stderr - -intelmq.bots.experts.tor_nodes.expert --update-database diff --git a/intelmq/bots/experts/tor_nodes/update-tor-nodes.license b/intelmq/bots/experts/tor_nodes/update-tor-nodes.license deleted file mode 100644 index 8fd09f5e0..000000000 --- a/intelmq/bots/experts/tor_nodes/update-tor-nodes.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2016 Sascha Wilde -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/setup.py b/setup.py index 88e1e3b25..32469384e 100644 --- a/setup.py +++ b/setup.py @@ -91,10 +91,6 @@ 'intelmq_psql_initdb = intelmq.bin.intelmq_psql_initdb:main', 'intelmq.bots.experts.sieve.validator = intelmq.bots.experts.sieve.validator:main', 'intelmqsetup = intelmq.bin.intelmqsetup:main', - 'update-asn-data = intelmq.bots.experts.asn_lookup.expert:BOT.update_database', - 'update-geoip-data = intelmq.bots.experts.maxmind_geoip.expert:BOT.update_database', - 'update-rfiprisk-data = intelmq.bots.experts.recordedfuture_iprisk.expert:BOT.update_database', - 'update-tor-nodes = intelmq.bots.experts.tor_nodes.expert:BOT.update_database', ] + BOTS, }, )