From 5abc9518bd219c74f00c334842b68ebf7b6ba5ed Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 20 Sep 2017 12:42:45 +0200 Subject: [PATCH 01/29] MAINT: lib/message: remove force parameter was deprecated for 1.0, this hasn't been deleted --- intelmq/lib/message.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/intelmq/lib/message.py b/intelmq/lib/message.py index 332c2b2a0..634e3c0e7 100644 --- a/intelmq/lib/message.py +++ b/intelmq/lib/message.py @@ -143,7 +143,7 @@ def is_valid(self, key: str, value: str, sanitize: bool=True) -> bool: return True return False - def add(self, key: str, value: str, sanitize: bool=True, force: bool=False, + def add(self, key: str, value: str, sanitize: bool=True, overwrite: bool=False, ignore: Sequence=(), raise_failure: bool=True) -> bool: """ @@ -170,7 +170,6 @@ def add(self, key: str, value: str, sanitize: bool=True, force: bool=False, intelmq.lib.exceptions.InvalidValue: If value is not valid for the given key and raise_failure is True. """ - overwrite = force or overwrite if not overwrite and key in self: raise exceptions.KeyExists(key) From 3068153295adfe3912cf1319ddedf950792b0222 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 20 Sep 2017 14:06:42 +0200 Subject: [PATCH 02/29] BUG: lib/splitreports: remove replaced force param --- intelmq/lib/splitreports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intelmq/lib/splitreports.py b/intelmq/lib/splitreports.py index 5b7ca0188..4130da941 100644 --- a/intelmq/lib/splitreports.py +++ b/intelmq/lib/splitreports.py @@ -149,7 +149,7 @@ def generate_reports(report_template: Report, infile: BinaryIO, chunk_size: Opti """ if chunk_size is None: report = report_template.copy() - report.add("raw", infile.read(), force=True) + report.add("raw", infile.read(), overwrite=True) yield report else: header = b"" @@ -157,5 +157,5 @@ def generate_reports(report_template: Report, infile: BinaryIO, chunk_size: Opti header = infile.readline() for chunk in read_delimited_chunks(infile, chunk_size): report = report_template.copy() - report.add("raw", header + chunk, force=True) + report.add("raw", header + chunk, overwrite=True) yield report From 7ea3faf625c40a14af6237e4ffc4bec7ae14fe98 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 8 Dec 2016 15:01:45 +0100 Subject: [PATCH 03/29] PKG: LSB paths for debian packages --- debian/conffiles | 10 +++++----- debian/cron.d/intelmq-update-data | 8 ++++---- debian/intelmq.postinst | 6 +++--- debian/patches/fix-dnspython-name.patch | 2 +- debian/rules | 14 ++++++++++---- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/debian/conffiles b/debian/conffiles index 37cee3ead..818133532 100644 --- a/debian/conffiles +++ b/debian/conffiles @@ -1,5 +1,5 @@ -/opt/intelmq/etc/harmonization.conf -/opt/intelmq/etc/BOTS -/opt/intelmq/etc/pipeline.conf -/opt/intelmq/etc/defaults.conf -/opt/intelmq/etc/runtime.conf +/etc/intelmq/harmonization.conf +/etc/intelmq/BOTS +/etc/intelmq/pipeline.conf +/etc/intelmq/defaults.conf +/etc/intelmq/runtime.conf diff --git a/debian/cron.d/intelmq-update-data b/debian/cron.d/intelmq-update-data index 380220427..cb907d45d 100644 --- a/debian/cron.d/intelmq-update-data +++ b/debian/cron.d/intelmq-update-data @@ -3,10 +3,10 @@ # # m h dom mon dow command # Update data for tor_nodes bot: -11 0 * * * intelmq /usr/bin/update-tor-nodes /opt/intelmq/var/lib/bots/tor_nodes/tor_nodes.dat +11 0 * * * intelmq /usr/bin/update-tor-nodes /var/lib/intelmq/bots/tor_nodes/tor_nodes.dat # Update data for maxmind_geoip bot: -17 0 * * * intelmq /usr/bin/update-geoip-data /opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb +17 0 * * * intelmq /usr/bin/update-geoip-data /var/lib/intelmq/bots/maxmind_geoip/GeoLite2-City.mmdb # Update data for asn_lookup bot: -23 0 * * * intelmq /usr/bin/update-asn-data /opt/intelmq/var/lib/bots/asn_lookup/ipasn.dat +23 0 * * * intelmq /usr/bin/update-asn-data /var/lib/intelmq/bots/asn_lookup/ipasn.dat # Update data for the RIPE DB abuse_c offline contact lookup -25 6 * * * intelmq /usr/bin/update-ripencc_abuse_contact_offline /opt/intelmq/var/lib/bots/ripencc_abuse_contact_offline/ +25 6 * * * intelmq /usr/bin/update-ripencc_abuse_contact_offline /var/lib/intelmq/bots/ripencc_abuse_contact_offline/ diff --git a/debian/intelmq.postinst b/debian/intelmq.postinst index 00399f128..22ae114a0 100644 --- a/debian/intelmq.postinst +++ b/debian/intelmq.postinst @@ -3,10 +3,10 @@ set -e if [ "$1" = "configure" ] ; then if ! getent passwd intelmq >/dev/null 2>&1; then - useradd -d /opt/intelmq -U -s /bin/bash intelmq + useradd -d /var/lib/intelmq -U -s /bin/bash intelmq fi - chmod -R g+w /opt/intelmq/* - chown -R intelmq:intelmq /opt/intelmq + chmod -R g+w /var/lib/intelmq/* + chown -R intelmq:intelmq /var/lib/intelmq fi #DEBHELPER# diff --git a/debian/patches/fix-dnspython-name.patch b/debian/patches/fix-dnspython-name.patch index 6d75d3a1d..d9bf32a9c 100644 --- a/debian/patches/fix-dnspython-name.patch +++ b/debian/patches/fix-dnspython-name.patch @@ -2,7 +2,7 @@ Description: Undo rename of the dependency dnspython3 The packages was renamed upstream; however, this change does not immediately affect this package's target distributions. Author: Gernot Schulz -Last-Update: 2016-07-20 +Last-Update: 2016-10-04 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/setup.py diff --git a/debian/rules b/debian/rules index 00b3084a0..90bb6e492 100755 --- a/debian/rules +++ b/debian/rules @@ -20,11 +20,17 @@ override_dh_auto_build: override_dh_auto_install: $(BOTDOCS) python3 setup.py install --root=debian/intelmq --prefix=/usr - mv debian/intelmq/opt/intelmq/etc/examples/* debian/intelmq/opt/intelmq/etc/ - rmdir debian/intelmq/opt/intelmq/etc/examples - mkdir -p debian/intelmq/opt/intelmq/var/log - mkdir -p debian/intelmq/opt/intelmq/var/lib/bots/file-output + mv debian/intelmq/etc/intelmq/examples/* debian/intelmq/etc/intelmq/ + rmdir debian/intelmq/etc/intelmq/examples + mkdir -p debian/intelmq/var/log/intelmq + mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output mkdir -p debian/intelmq/etc/logrotate.d + # Install a modifier bot config file based on an example + cp -a debian/intelmq/var/lib/intelmq/bots/modify/example/default.conf \ + debian/intelmq/var/lib/intelmq/bots/modify/modify.conf + # Remove modify bot example configs from their original location because + # intelmq.install copies them to the /usr/share/doc/intelmq/bots hierarchy + rm -vr debian/intelmq/var/lib/intelmq/bots/modify/example ## BOTS # Include all bot READMEs for readme in $(foreach bot,$(BOTDOCS),$(subst intelmq/bots/,,$(bot))); \ From efeb44366cf862d2d5e910f292115688850037b3 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 8 Dec 2016 15:33:28 +0100 Subject: [PATCH 04/29] PKG: switch debian package to quilt --- debian/source/format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/source/format b/debian/source/format index 89ae9db8f..163aaf8d8 100644 --- a/debian/source/format +++ b/debian/source/format @@ -1 +1 @@ -3.0 (native) +3.0 (quilt) From 277b9b3f8ac059bdec25e5baa49b3950089476bf Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 8 Dec 2016 16:02:38 +0100 Subject: [PATCH 05/29] PKG: replace paths in docs with sed --- debian/rules | 1 + debian/sedfile | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 debian/sedfile diff --git a/debian/rules b/debian/rules index 90bb6e492..b70927d99 100755 --- a/debian/rules +++ b/debian/rules @@ -19,6 +19,7 @@ export PYBUILD_NAME=intelmq override_dh_auto_build: override_dh_auto_install: $(BOTDOCS) + find . -type f -exec sed -i -f debian/sedfile {} \; python3 setup.py install --root=debian/intelmq --prefix=/usr mv debian/intelmq/etc/intelmq/examples/* debian/intelmq/etc/intelmq/ rmdir debian/intelmq/etc/intelmq/examples diff --git a/debian/sedfile b/debian/sedfile new file mode 100644 index 000000000..5fd784fe4 --- /dev/null +++ b/debian/sedfile @@ -0,0 +1,5 @@ +s/opt\/intelmq\/var\/run/var\/run\/intelmq/g +s/opt\/intelmq\/var\/log/var\/log\/intelmq/g +s/opt\/intelmq\/var\/lib/var\/lib\/intelmq/g +s/opt\/intelmq\/etc\//etc\/intelmq\//g +s/opt\/intelmq/etc\/intelmq/g From 73765eb82c6b3f60308dfcb12274b255d6437bf9 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 8 Dec 2016 17:57:35 +0100 Subject: [PATCH 06/29] PKG: make debian rules file complete --- debian/control | 2 +- debian/intelmq.postinst | 7 +++++-- debian/intelmq.tmpfile | 1 + debian/rules | 24 ++++++++++++++++++++---- 4 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 debian/intelmq.tmpfile diff --git a/debian/control b/debian/control index ff6148262..d8cfbeb2e 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: intelmq Maintainer: Sascha Wilde Section: python Priority: optional -Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt +Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt, dh-systemd X-Python3-Version: >= 3.3.0 Standards-Version: 3.9.6 Homepage: https://github.com/certtools/intelmq/ diff --git a/debian/intelmq.postinst b/debian/intelmq.postinst index 22ae114a0..d4c417f1e 100644 --- a/debian/intelmq.postinst +++ b/debian/intelmq.postinst @@ -2,11 +2,14 @@ set -e if [ "$1" = "configure" ] ; then + if ! getent group intelmq >/dev/null 2>&1; then + groupadd -r intelmq + fi if ! getent passwd intelmq >/dev/null 2>&1; then - useradd -d /var/lib/intelmq -U -s /bin/bash intelmq + useradd -r -g intelmq -d /var/lib/intelmq/ -c "user running intelmq" -U -s /bin/bash intelmq fi chmod -R g+w /var/lib/intelmq/* - chown -R intelmq:intelmq /var/lib/intelmq + chown -R intelmq:intelmq /etc/intelmq/ /var/lib/intelmq/ /var/log/intelmq/ fi #DEBHELPER# diff --git a/debian/intelmq.tmpfile b/debian/intelmq.tmpfile new file mode 100644 index 000000000..08adca7d2 --- /dev/null +++ b/debian/intelmq.tmpfile @@ -0,0 +1 @@ +D /run/intelmq 0755 intelmq intelmq diff --git a/debian/rules b/debian/rules index b70927d99..c25c33036 100755 --- a/debian/rules +++ b/debian/rules @@ -6,7 +6,7 @@ DOCS_DIR := debian/intelmq/usr/share/doc/intelmq BOTROOT := intelmq/bots BOTCATEGORIES := collectors experts outputs parsers BOTCATEGORIES := $(foreach cat,$(BOTCATEGORIES),$(BOTROOT)/$(cat)) -BOTS := $(foreach bot,$(BOTCATEGORIES),$(wildcard $(bot)/*)) +BOTS := $(foreach bot,$(BOTCATEGORIES),$(wildcard $(bot)/*)) BOTDOCS := $(foreach bot,$(BOTS),$(wildcard $(bot)/*.md)) # This file is based on a version @@ -14,15 +14,28 @@ BOTDOCS := $(foreach bot,$(BOTS),$(wildcard $(bot)/*.md)) # Wed, 23 Mar 2016 17:49:26 +0000 export PYBUILD_NAME=intelmq %: - dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt + dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd + #dh_installinit # breaks override_dh_auto_build: override_dh_auto_install: $(BOTDOCS) + rm intelmq/bin/rewrite_config_files.py + rm intelmq/bin/intelmq_gen_harm_docs.py + rm -r docs/install-scripts/ + sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py find . -type f -exec sed -i -f debian/sedfile {} \; python3 setup.py install --root=debian/intelmq --prefix=/usr - mv debian/intelmq/etc/intelmq/examples/* debian/intelmq/etc/intelmq/ - rmdir debian/intelmq/etc/intelmq/examples + # these are already in /usr/bin/ + #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/maxmind_geoip/update-geoip-data + #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/asn_lookup/update-asn-data + #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/tor_nodes/update-tor-nodes + # and rename those in /usr/bin + mv debian/intelmq/usr/bin/update-geoip-data debian/intelmq/usr/bin/intelmq-update-geoip-data + mv debian/intelmq/usr/bin/update-asn-data debian/intelmq/usr/bin/intelmq-update-asn-data + mv debian/intelmq/usr/bin/update-tor-nodes debian/intelmq/usr/bin/intelmq-update-tor-nodes + # install testdata + # find intelmq/tests/ -type f ! -name "*.py" ! -name "*.pyc" | xargs -I '{}' cp --parents '{}' %{buildroot}/%{python3_sitelib}/ mkdir -p debian/intelmq/var/log/intelmq mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output mkdir -p debian/intelmq/etc/logrotate.d @@ -39,3 +52,6 @@ override_dh_auto_install: $(BOTDOCS) mkdir -p $(DOCS_DIR)/bots/$$(dirname $$readme); \ cp intelmq/bots/$$readme $(DOCS_DIR)/bots/$$(dirname $$readme); \ done + +override_dh_systemd_start: + dh_systemd_start --restart-after-upgrade From 392a63eb071f2b605d592b352f307c9f0d3cfe54 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 9 Dec 2016 21:24:30 +0100 Subject: [PATCH 07/29] PKG: fix modify config bugs, install example conf --- debian/intelmq.postinst | 2 +- debian/rules | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/debian/intelmq.postinst b/debian/intelmq.postinst index d4c417f1e..af8c62f02 100644 --- a/debian/intelmq.postinst +++ b/debian/intelmq.postinst @@ -6,7 +6,7 @@ if [ "$1" = "configure" ] ; then groupadd -r intelmq fi if ! getent passwd intelmq >/dev/null 2>&1; then - useradd -r -g intelmq -d /var/lib/intelmq/ -c "user running intelmq" -U -s /bin/bash intelmq + useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -U -s /bin/bash intelmq fi chmod -R g+w /var/lib/intelmq/* chown -R intelmq:intelmq /etc/intelmq/ /var/lib/intelmq/ /var/log/intelmq/ diff --git a/debian/rules b/debian/rules index c25c33036..72ffdb065 100755 --- a/debian/rules +++ b/debian/rules @@ -26,6 +26,8 @@ override_dh_auto_install: $(BOTDOCS) sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py find . -type f -exec sed -i -f debian/sedfile {} \; python3 setup.py install --root=debian/intelmq --prefix=/usr + mv debian/intelmq/etc/intelmq/examples/* debian/intelmq/etc/intelmq/ + rmdir debian/intelmq/etc/intelmq/examples # these are already in /usr/bin/ #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/maxmind_geoip/update-geoip-data #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/asn_lookup/update-asn-data @@ -39,12 +41,7 @@ override_dh_auto_install: $(BOTDOCS) mkdir -p debian/intelmq/var/log/intelmq mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output mkdir -p debian/intelmq/etc/logrotate.d - # Install a modifier bot config file based on an example - cp -a debian/intelmq/var/lib/intelmq/bots/modify/example/default.conf \ - debian/intelmq/var/lib/intelmq/bots/modify/modify.conf - # Remove modify bot example configs from their original location because - # intelmq.install copies them to the /usr/share/doc/intelmq/bots hierarchy - rm -vr debian/intelmq/var/lib/intelmq/bots/modify/example + mkdir -p debian/intelmq/var/lib/intelmq/bots/modify/ ## BOTS # Include all bot READMEs for readme in $(foreach bot,$(BOTDOCS),$(subst intelmq/bots/,,$(bot))); \ From e044595c84955a1a642b123b7de51243170cb609 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 9 Dec 2016 22:23:51 +0100 Subject: [PATCH 08/29] PKG: fix user creation --- debian/intelmq.postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/intelmq.postinst b/debian/intelmq.postinst index af8c62f02..25d54594c 100644 --- a/debian/intelmq.postinst +++ b/debian/intelmq.postinst @@ -6,7 +6,7 @@ if [ "$1" = "configure" ] ; then groupadd -r intelmq fi if ! getent passwd intelmq >/dev/null 2>&1; then - useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -U -s /bin/bash intelmq + useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq fi chmod -R g+w /var/lib/intelmq/* chown -R intelmq:intelmq /etc/intelmq/ /var/lib/intelmq/ /var/log/intelmq/ From fb8c3e23428e2253cdc8f78bcb464af27483342a Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 5 Apr 2017 19:05:22 +0200 Subject: [PATCH 09/29] PKG: patch intelmq internal paths --- debian/patches/fix-intelmq-paths.patch | 24 ++++++++++++++++++++++++ debian/patches/series | 1 + 2 files changed, 25 insertions(+) create mode 100644 debian/patches/fix-intelmq-paths.patch diff --git a/debian/patches/fix-intelmq-paths.patch b/debian/patches/fix-intelmq-paths.patch new file mode 100644 index 000000000..0570e6bf5 --- /dev/null +++ b/debian/patches/fix-intelmq-paths.patch @@ -0,0 +1,24 @@ +Description: Set all paths to LSB +Author: Sebastian Wagner +Last-Update: 2017-09-20 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/intelmq/__init__.py ++++ b/intelmq/__init__.py +@@ -1,13 +1,12 @@ + from .version import __version__ + import os + +-ROOT_DIR = "/opt/intelmq/" +-CONFIG_DIR = os.path.join(ROOT_DIR, "etc/") ++CONFIG_DIR = "/etc/intelmq/" + DEFAULT_LOGGING_LEVEL = "INFO" + BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS") +-DEFAULT_LOGGING_PATH = os.path.join(ROOT_DIR, "var/log/") ++DEFAULT_LOGGING_PATH = "/var/log/intelmq/" + DEFAULTS_CONF_FILE = os.path.join(CONFIG_DIR, "defaults.conf") + HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf") + PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf") + RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf") +-VAR_RUN_PATH = os.path.join(ROOT_DIR, "var/run/") ++VAR_RUN_PATH = "/var/run/intelmq/" diff --git a/debian/patches/series b/debian/patches/series index f51972d39..081308755 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,2 +1,3 @@ fix-logrotate-path.patch fix-dnspython-name.patch +fix-intelmq-paths.patch From 760a75a429f7b1eae6fe30c2614be90c8215d32d Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 10 Apr 2017 15:25:05 +0200 Subject: [PATCH 10/29] PKG: update debian contro, lower dependencies --- debian/control | 11 +++++------ debian/rules | 3 +-- setup.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/debian/control b/debian/control index d8cfbeb2e..283b98408 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: intelmq Maintainer: Sascha Wilde Section: python Priority: optional -Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt, dh-systemd +Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt, dh-systemd, python3-typing X-Python3-Version: >= 3.3.0 Standards-Version: 3.9.6 Homepage: https://github.com/certtools/intelmq/ @@ -10,14 +10,13 @@ Homepage: https://github.com/certtools/intelmq/ Package: intelmq Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, - libffi-dev, python3-dateutil (>= 2.0), python3-dnspython (>= 1.11.1), - python3-pymongo, python3-openssl, python3-psutil (>= 1.2.1), - python3-psycopg2, python3-redis (>= 2.10.3), + libffi-dev, python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1), + python3-openssl, python3-psutil (>= 1.2.1), python3-redis (>= 2.10), python3-requests (>= 2.2.1), python3-termstyle (>= 0.1.10), python3-tz, - redis-server, cron, bash-completion, jq + redis-server, cron, bash-completion, jq, python3-typing Recommends: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0), python3-stomp.py (>= 4.1.9), python3-sleekxmpp (>= 1.3.1), - python3-geoip2 (>= 2.2.0) + python3-geoip2 (>= 2.2.0), python3-pymongo, python3-psycopg2 Description: IntelMQ is a solution for IT security teams (CERTs, CSIRTs, abuse departments,...) for collecting and processing security feeds (such as log files) using a message queuing protocol. It's a community driven initiative diff --git a/debian/rules b/debian/rules index 72ffdb065..156e3003c 100755 --- a/debian/rules +++ b/debian/rules @@ -36,8 +36,7 @@ override_dh_auto_install: $(BOTDOCS) mv debian/intelmq/usr/bin/update-geoip-data debian/intelmq/usr/bin/intelmq-update-geoip-data mv debian/intelmq/usr/bin/update-asn-data debian/intelmq/usr/bin/intelmq-update-asn-data mv debian/intelmq/usr/bin/update-tor-nodes debian/intelmq/usr/bin/intelmq-update-tor-nodes - # install testdata - # find intelmq/tests/ -type f ! -name "*.py" ! -name "*.pyc" | xargs -I '{}' cp --parents '{}' %{buildroot}/%{python3_sitelib}/ + # create directories mkdir -p debian/intelmq/var/log/intelmq mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output mkdir -p debian/intelmq/etc/logrotate.d diff --git a/setup.py b/setup.py index 285f04638..ed4f985ae 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ 'python-dateutil>=2.5', 'python-termstyle>=0.1.10', 'pytz>=2014.1', - 'redis>=2.10.3', + 'redis>=2.10', 'requests>=2.2.0', ] if sys.version_info < (3, 5): From f15a2961248d7168434d0abd01fb81d60ee52ad3 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 11 Apr 2017 15:00:19 +0200 Subject: [PATCH 11/29] PKG: debian package fixes --- debian/control | 4 ++-- debian/intelmq.postinst | 1 + debian/intelmq.postrm | 7 ++++++- setup.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/debian/control b/debian/control index 283b98408..40a48e47b 100644 --- a/debian/control +++ b/debian/control @@ -10,11 +10,11 @@ Homepage: https://github.com/certtools/intelmq/ Package: intelmq Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, - libffi-dev, python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1), + python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1), python3-openssl, python3-psutil (>= 1.2.1), python3-redis (>= 2.10), python3-requests (>= 2.2.1), python3-termstyle (>= 0.1.10), python3-tz, redis-server, cron, bash-completion, jq, python3-typing -Recommends: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0), +Suggests: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0), python3-stomp.py (>= 4.1.9), python3-sleekxmpp (>= 1.3.1), python3-geoip2 (>= 2.2.0), python3-pymongo, python3-psycopg2 Description: IntelMQ is a solution for IT security teams (CERTs, CSIRTs, abuse diff --git a/debian/intelmq.postinst b/debian/intelmq.postinst index 25d54594c..99d6fbcdf 100644 --- a/debian/intelmq.postinst +++ b/debian/intelmq.postinst @@ -10,6 +10,7 @@ if [ "$1" = "configure" ] ; then fi chmod -R g+w /var/lib/intelmq/* chown -R intelmq:intelmq /etc/intelmq/ /var/lib/intelmq/ /var/log/intelmq/ + systemd-tmpfiles --create /usr/lib/tmpfiles.d/intelmq.conf fi #DEBHELPER# diff --git a/debian/intelmq.postrm b/debian/intelmq.postrm index 2daebfb1c..48a88716e 100644 --- a/debian/intelmq.postrm +++ b/debian/intelmq.postrm @@ -2,7 +2,12 @@ set -e if [ "$1" = "remove" ] ; then - deluser intelmq + if getent group intelmq >/dev/null 2>&1; then + groupdel -r intelmq + fi + if getent passwd intelmq >/dev/null 2>&1; then + userdel intelmq + fi fi #DEBHELPER# diff --git a/setup.py b/setup.py index ed4f985ae..dd919de0e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ 'psutil>=1.2.1', 'python-dateutil>=2.5', 'python-termstyle>=0.1.10', - 'pytz>=2014.1', + 'pytz>=2012c', 'redis>=2.10', 'requests>=2.2.0', ] From ddc897626316503c36b61f477580af293c4e0142 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 2 Jun 2017 12:47:00 +0200 Subject: [PATCH 12/29] PKG/BUG: install-scripts have been removed --- debian/rules | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/rules b/debian/rules index 156e3003c..7a9df33ca 100755 --- a/debian/rules +++ b/debian/rules @@ -22,7 +22,6 @@ override_dh_auto_build: override_dh_auto_install: $(BOTDOCS) rm intelmq/bin/rewrite_config_files.py rm intelmq/bin/intelmq_gen_harm_docs.py - rm -r docs/install-scripts/ sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py find . -type f -exec sed -i -f debian/sedfile {} \; python3 setup.py install --root=debian/intelmq --prefix=/usr From ec851905fc6e66233b3cf716c77364be3b94965f Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 14 Jun 2017 17:15:33 +0200 Subject: [PATCH 13/29] PKG: remove typing from debian requirements It's only needed for Debian 8, add it later in the build process --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 40a48e47b..497c2a864 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: intelmq Maintainer: Sascha Wilde Section: python Priority: optional -Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt, dh-systemd, python3-typing +Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt, dh-systemd X-Python3-Version: >= 3.3.0 Standards-Version: 3.9.6 Homepage: https://github.com/certtools/intelmq/ @@ -13,7 +13,7 @@ Depends: ${misc:Depends}, ${python3:Depends}, python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1), python3-openssl, python3-psutil (>= 1.2.1), python3-redis (>= 2.10), python3-requests (>= 2.2.1), python3-termstyle (>= 0.1.10), python3-tz, - redis-server, cron, bash-completion, jq, python3-typing + redis-server, cron, bash-completion, jq Suggests: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0), python3-stomp.py (>= 4.1.9), python3-sleekxmpp (>= 1.3.1), python3-geoip2 (>= 2.2.0), python3-pymongo, python3-psycopg2 From 8a0371ad7a2b4529cefcb9623667e316d8746c0c Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 20 Jun 2017 16:06:06 +0200 Subject: [PATCH 14/29] PKG: changelog entry for new revision --- debian/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debian/changelog b/debian/changelog index b8623da31..639b46329 100644 --- a/debian/changelog +++ b/debian/changelog @@ -20,6 +20,10 @@ intelmq (1.0.0.rc1-1) experimental; urgency=medium * update to version 1.0.0.rc1 +intelmq (1.0.0.dev8-2) experimental; urgency=medium + + * packaging fixes + -- Sebastian Wagner Wed, 20 Jun 2017 16:05:00 +0200 intelmq (1.0.0.dev8-1) experimental; urgency=medium From e0b21ee69d972b575b5e48a189877e1dd2dcbba1 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 6 Jul 2017 12:36:21 +0200 Subject: [PATCH 15/29] PKG: Add systemd as dependency Required for systemd-tmpfiles --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 497c2a864..5ac34da1c 100644 --- a/debian/control +++ b/debian/control @@ -13,7 +13,7 @@ Depends: ${misc:Depends}, ${python3:Depends}, python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1), python3-openssl, python3-psutil (>= 1.2.1), python3-redis (>= 2.10), python3-requests (>= 2.2.1), python3-termstyle (>= 0.1.10), python3-tz, - redis-server, cron, bash-completion, jq + redis-server, cron, bash-completion, jq, systemd Suggests: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0), python3-stomp.py (>= 4.1.9), python3-sleekxmpp (>= 1.3.1), python3-geoip2 (>= 2.2.0), python3-pymongo, python3-psycopg2 From db652d103a1922f81ffaeb89af91b13ce6eb05a2 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 6 Jul 2017 14:49:15 +0200 Subject: [PATCH 16/29] TST: travis: install dh-systemd for package builds --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5fa67dd1a..3828937f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ python: - "3.5" - "3.6" install: - - sudo apt-get install dpkg-dev dh-python python-setuptools python3-setuptools python3-all debhelper quilt polipo lighttpd fakeroot + - sudo apt-get install dpkg-dev dh-python python-setuptools python3-setuptools python3-all debhelper quilt polipo lighttpd fakeroot dh-systemd - if [[ $REQUIREMENTS == true ]] ; then for file in intelmq/bots/*/*/REQUIREMENTS.txt; do pip install -r $file; done; fi - if [[ $TRAVIS_PYTHON_VERSION < '3.5' ]]; then pip install typing; fi - pip install codecov From 92628b70e1b5273053c23e1ca8104633cde61a14 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 6 Jul 2017 19:15:36 +0200 Subject: [PATCH 17/29] TST: travis: fix package build with quilt --- .travis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3828937f1..504aaa0d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,19 @@ before_script: - intelmq_psql_initdb - sed -i 's/events/tests/g' /tmp/initdb.sql - psql -v ON_ERROR_STOP=on -f /tmp/initdb.sql intelmq -U intelmq + - VERSION=$(git describe --abbrev=0 --tags) + - git archive --format=tar.gz HEAD > ../intelmq_$VERSION.orig.tar.gz + - git archive --format=tar.gz --prefix=debian/ HEAD:debian/ > ../intelmq_$VERSION-1.debian.tar.gz + - pushd .. + - mkdir build + - cd build + - tar -xzf ../intelmq_$VERSION.orig.tar.gz + - tar -xzf ../intelmq_$VERSION-1.debian.tar.gz + - popd script: - if [[ $REQUIREMENTS == true ]] ; then INTELMQ_TEST_DATABASES=1 INTELMQ_TEST_LOCAL_WEB=1 INTELMQ_TEST_EXOTIC=1 nosetests --with-coverage --cover-package=intelmq --cover-branches; else INTELMQ_TEST_LOCAL_WEB=1 nosetests --with-coverage --cover-package=intelmq --cover-branches; fi - - dpkg-buildpackage -us -uc - pycodestyle intelmq/{bots,lib,bin} + - pushd ../build && dpkg-buildpackage -us -uc; popd services: - redis-server - postgresql From 02378b6d6470adf49d179e02483257cc265b21cc Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 31 Jul 2017 10:32:29 +0200 Subject: [PATCH 18/29] DOC: Update install instructions --- docs/INSTALL.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 5b7648d1c..3d119d233 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -111,11 +111,11 @@ https://software.opensuse.org/download.html?project=home%3Asebix%3Aintelmq&packa Currently, these operating systems are supported by the packages: * CentOS 7, install `epel-release` first -* Debian 8, install `python3-typing` too -* Fedora 25 -* openSUSE Leap 42.2 and 42.3 -* openSUSE Tumbleweed -* Ubuntu 16.04 +* RHEL 7, install `epel-release` first +* Debian 8 (install `python3-typing` too) and 9 +* Fedora 25, 26 and Rawhide +* openSUSE Leap 42.2, 42.3 and Tumbleweed +* Ubuntu 16.04 and 17.04 Please report any errors or improvements at https://github.com/certtools/intelmq/issues Thanks! @@ -158,6 +158,7 @@ sudo -s pip3 install . +mkdir /opt/intelmq useradd -d /opt/intelmq -U -s /bin/bash intelmq chmod -R 0770 /opt/intelmq chown -R intelmq.intelmq /opt/intelmq From 67ac8885c5d6940e1d2c118109d7a386c640ba47 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 4 Aug 2017 08:34:36 +0200 Subject: [PATCH 19/29] PKG: Fix build on newer debians and ubuntus --- debian/control | 4 ++-- debian/patches/series | 1 - debian/rules | 10 +++++++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/debian/control b/debian/control index 5ac34da1c..0d52ab9cb 100644 --- a/debian/control +++ b/debian/control @@ -2,8 +2,8 @@ Source: intelmq Maintainer: Sascha Wilde Section: python Priority: optional -Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt, dh-systemd -X-Python3-Version: >= 3.3.0 +Build-Depends: debhelper (>= 4.1.16), python3-all, python3-setuptools, quilt, dh-python, python-setuptools, dh-systemd, safe-rm, python3-requests, python3-redis, python3-dnspython, python3-psutil, python3-dateutil, python3-termstyle, python3-tz, lsb-release +X-Python3-Version: >= 3.3 Standards-Version: 3.9.6 Homepage: https://github.com/certtools/intelmq/ diff --git a/debian/patches/series b/debian/patches/series index 081308755..1ffad34fd 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,3 +1,2 @@ fix-logrotate-path.patch -fix-dnspython-name.patch fix-intelmq-paths.patch diff --git a/debian/rules b/debian/rules index 7a9df33ca..6a13b3b91 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,9 @@ #!/usr/bin/make -f +CODENAME := $(shell lsb_release -cs) DOCS_DIR := debian/intelmq/usr/share/doc/intelmq +export INTELMQ_SKIP_REDIS=1 +export INTELMQ_SKIP_INTERNET=1 # Compile list of bot READMEs BOTROOT := intelmq/bots @@ -15,7 +18,12 @@ BOTDOCS := $(foreach bot,$(BOTS),$(wildcard $(bot)/*.md)) export PYBUILD_NAME=intelmq %: dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd - #dh_installinit # breaks + +build: + if [ $(CODENAME) = 'xenial' ] || [ $(CODENAME) = 'jessie' ]; then\ + patch -p1 setup.py debian/patches/fix-dnspython-name.patch;\ + fi + dh build --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd override_dh_auto_build: From 5cedbdc5ecb18beefc37a1258ff2d338dc129895 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 10 Jul 2017 17:03:26 +0200 Subject: [PATCH 20/29] TST: Fix pipeline PythonList tests (workaround) there's bug with non-deleted queues, this is a workaround only happens when starting the tests with unittest itself (i.e. python3 setup.py test) --- intelmq/tests/lib/test_pipeline.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/intelmq/tests/lib/test_pipeline.py b/intelmq/tests/lib/test_pipeline.py index d7420b7d1..70a94eebe 100644 --- a/intelmq/tests/lib/test_pipeline.py +++ b/intelmq/tests/lib/test_pipeline.py @@ -5,7 +5,6 @@ We are testing sending and receiving on the same queue for Redis and Pythonlist. TODO: clear_queues -TODO: count_queued_messages TODO: acknowledge TODO: check internal representation of data in redis (like with Pythonlist) """ @@ -31,39 +30,43 @@ def setUp(self): params = Parameters() params.broker = 'Pythonlist' self.pipe = pipeline.PipelineFactory.create(params) - self.pipe.set_queues('src', 'source') - self.pipe.set_queues('dst', 'destination') + self.pipe.set_queues('test-bot-input', 'source') + self.pipe.set_queues('test-bot-output', 'destination') def test_receive(self): - self.pipe.state['src'] = [SAMPLES['normal'][0]] + self.pipe.state['test-bot-input'] = [SAMPLES['normal'][0]] self.assertEqual(SAMPLES['normal'][1], self.pipe.receive()) def test_send(self): self.pipe.send(SAMPLES['normal'][1]) self.assertEqual(SAMPLES['normal'][0], - self.pipe.state['dst'][0]) + self.pipe.state['test-bot-output'][0]) def test_receive_unicode(self): - self.pipe.state['src'] = [SAMPLES['unicode'][0]] + self.pipe.state['test-bot-input'] = [SAMPLES['unicode'][0]] self.assertEqual(SAMPLES['unicode'][1], self.pipe.receive()) def test_send_unicode(self): self.pipe.send(SAMPLES['unicode'][1]) self.assertEqual(SAMPLES['unicode'][0], - self.pipe.state['dst'][0]) + self.pipe.state['test-bot-output'][0]) def test_count(self): self.pipe.send(SAMPLES['normal'][0]) self.pipe.send(SAMPLES['normal'][1]) self.pipe.send(SAMPLES['unicode'][0]) - self.assertEqual(self.pipe.count_queued_messages('dst'), {'dst': 3}) + self.assertEqual(self.pipe.count_queued_messages('test-bot-output'), + {'test-bot-output': 3}) def test_count_multi(self): - self.pipe.state['src'] = [SAMPLES['normal'][0]] + self.pipe.state['test-bot-input'] = [SAMPLES['normal'][0]] self.pipe.send(SAMPLES['normal'][0]) self.pipe.send(SAMPLES['unicode'][0]) - self.assertEqual(self.pipe.count_queued_messages('src', 'dst'), - {'src': 1, 'dst': 2}) + self.assertEqual(self.pipe.count_queued_messages('test-bot-input', 'test-bot-output'), + {'test-bot-input': 1, 'test-bot-output': 2}) + + def tearDown(self): + self.pipe.state = {} @test.skip_redis() From 10e716d6b4a7cece6e05f9ead9d70402eafbb1d7 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 20 Sep 2017 16:27:28 +0200 Subject: [PATCH 21/29] PKG: new revision for base branch change --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 639b46329..e1a492156 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +intelmq (1.0.1-2) experimental; urgency=low + + * New base branch 'packaging' for packages + + -- Wagner Sebastian Wed, 20 Sep 2017 16:26:59 +0200 + intelmq (1.0.1-1) experimental; urgency=low * update to version 1.0.1 From 0283a45a3e95c3950ce5d42cb88e5bee477e0781 Mon Sep 17 00:00:00 2001 From: Edvard Rejthar Date: Fri, 29 Sep 2017 19:39:17 +0200 Subject: [PATCH 22/29] deduplication bypass default I think, the default value should be False (as seen in the code at https://github.com/certtools/intelmq/blob/master/intelmq/bots/experts/deduplicator/expert.py line 41) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d57483513..b9bb4dd89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ CHANGELOG - `bots.parsers.alienvault.parser_otx`: handle timestamps without floating point seconds ### Experts -- bots.experts.deduplicator: New parameter `bypass` to deactivate deduplication, default: true +- bots.experts.deduplicator: New parameter `bypass` to deactivate deduplication, default: False v1.0.0.dev8 ----------- From 82f4eec9526a42e2359290f885cbcee921426820 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 9 Oct 2017 16:03:25 +0200 Subject: [PATCH 23/29] BUG: coll/mail/url: reconnect for marking as seen if necessary fixes certtools/intelmq#852 --- CHANGELOG.md | 6 ++++++ intelmq/bots/collectors/mail/collector_mail_url.py | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9bb4dd89..20ce36327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ CHANGELOG ========== +1.0.2 Bugfix release +-------------------- + +### Bots +- `collectors.mail.collector_mail_url`: Fix bug which prevented marking emails seen due to disconnects from server (#852). + 1.0.1 Bugfix release -------------------- ### Documentation diff --git a/intelmq/bots/collectors/mail/collector_mail_url.py b/intelmq/bots/collectors/mail/collector_mail_url.py index dd2ce18f3..5ccf991af 100755 --- a/intelmq/bots/collectors/mail/collector_mail_url.py +++ b/intelmq/bots/collectors/mail/collector_mail_url.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re import io +import imaplib import requests try: @@ -26,11 +27,15 @@ def init(self): self.chunk_replicate_header = getattr(self.parameters, 'chunk_replicate_header', None) - def process(self): + def connect_mailbox(self): mailbox = imbox.Imbox(self.parameters.mail_host, self.parameters.mail_user, self.parameters.mail_password, self.parameters.mail_ssl) + return mailbox + + def process(self): + mailbox = self.connect_mailbox() emails = mailbox.messages(folder=self.parameters.folder, unread=True) if emails: @@ -90,7 +95,12 @@ def process(self): # Only mark read if message relevant to this instance, # so other instances watching this mailbox will still # check it. - mailbox.mark_seen(uid) + try: + mailbox.mark_seen(uid) + except imaplib.abort: + # Disconnect, see https://github.com/certtools/intelmq/issues/852 + mailbox = self.connect_mailbox() + mailbox.mark_seen(uid) if not erroneous: self.logger.info("Email report read.") From 2ee82fe46b349abce5f7f863e01c0711f0756e40 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 9 Oct 2017 16:18:11 +0200 Subject: [PATCH 24/29] Revert "Revert "TST: cymru: fix changed ASN"" This reverts commit d8f878a4120b7f6e5beb30ba11b88d65f127d861. --- intelmq/tests/bots/experts/cymru_whois/test_expert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intelmq/tests/bots/experts/cymru_whois/test_expert.py b/intelmq/tests/bots/experts/cymru_whois/test_expert.py index bb2504d6a..11e1c1761 100644 --- a/intelmq/tests/bots/experts/cymru_whois/test_expert.py +++ b/intelmq/tests/bots/experts/cymru_whois/test_expert.py @@ -71,8 +71,8 @@ EXAMPLE_6TO4_OUTPUT = {"__type": "Event", "source.ip": "2002:3ee0:3972:0001::1", "source.network": "2002::/16", - "source.asn": 6939, - "source.as_name": "HURRICANE - Hurricane Electric, Inc., US", + "source.asn": 1103, + "source.as_name": "SURFNET-NL SURFnet, The Netherlands, NL", "time.observation": "2015-01-01T00:00:00+00:00", } From 0423854e5416f7e57f88b97f5d4edcdf56750840 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 6 Nov 2017 11:28:54 +0100 Subject: [PATCH 25/29] BUG: spamhaus cert parser: handle AS? fixes certtools/intelmq#1111 --- intelmq/bots/parsers/spamhaus/parser_cert.py | 5 ++++- intelmq/tests/bots/parsers/spamhaus/cert.txt | 1 + .../tests/bots/parsers/spamhaus/test_parser_cert.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/spamhaus/parser_cert.py b/intelmq/bots/parsers/spamhaus/parser_cert.py index 66171eddb..d74d7dd35 100644 --- a/intelmq/bots/parsers/spamhaus/parser_cert.py +++ b/intelmq/bots/parsers/spamhaus/parser_cert.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """ +Header of the File: ; Bots filtered by last 1 hours, prepared for on UTC = ... ; Copyright © 2015 The Spamhaus Project Ltd. All rights reserved. ; No re-distribution or public access allowed without Spamhaus permission. @@ -47,7 +48,9 @@ def process(self): event.change("feed.url", event["feed.url"].split("key=")[0]) event.add('source.ip', row_splitted[0]) - event.add('source.asn', row_splitted[1].replace('AS', '')) + source_asn = row_splitted[1].replace('AS', '') + if source_asn != '?': + event.add('source.asn', source_asn) event.add('source.geolocation.cc', row_splitted[2]) event.add('time.source', DateTime.from_timestamp(int(row_splitted[3]))) diff --git a/intelmq/tests/bots/parsers/spamhaus/cert.txt b/intelmq/tests/bots/parsers/spamhaus/cert.txt index 4ed3a87b9..c46c8d524 100644 --- a/intelmq/tests/bots/parsers/spamhaus/cert.txt +++ b/intelmq/tests/bots/parsers/spamhaus/cert.txt @@ -4,3 +4,4 @@ 109.91.0.227,AS6830,AT,1441011657,conficker,216.66.15.109,216.66.15.109,80,1430,tcp 111.111.111.183,AS11178,LV,1471111139,iotmirai,-,?,?,?,? 111.111.111.230,AS11178,LV,1471111134,gootkit,,111.111.111.166,1696,xxxxxxxxxxx.com,tcp +203.0.113.04,AS?,AT,1509955710,locky,example.invalid,198.18.0.19,80,36288,tcp diff --git a/intelmq/tests/bots/parsers/spamhaus/test_parser_cert.py b/intelmq/tests/bots/parsers/spamhaus/test_parser_cert.py index 05c1435e6..b0533b5d9 100644 --- a/intelmq/tests/bots/parsers/spamhaus/test_parser_cert.py +++ b/intelmq/tests/bots/parsers/spamhaus/test_parser_cert.py @@ -9,6 +9,7 @@ with open(os.path.join(os.path.dirname(__file__), 'cert.txt')) as handle: FILE = handle.read() +FILE_LINES = FILE.splitlines() EXAMPLE_REPORT = {"feed.url": "https://portal.spamhaus.org/cert/api.php?cert=" "&key=", @@ -78,6 +79,17 @@ 'destination.port': 1696, 'source.geolocation.cc': 'LV', 'protocol.transport': 'tcp', + }, + {'raw': utils.base64_encode(FILE_LINES[-1]), + 'source.ip': '203.0.113.4', + 'time.source': '2017-11-06T08:08:30+00:00', + 'malware.name': 'locky', + 'destination.ip': '198.18.0.19', + 'destination.fqdn': 'example.invalid', + 'destination.port': 80, + 'source.geolocation.cc': 'AT', + 'protocol.transport': 'tcp', + 'extra': '{"destination.local_port": 36288}', }] From 437b0b6ca32d95a7516d27e08b71e69bd01bfc59 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 6 Nov 2017 12:02:31 +0100 Subject: [PATCH 26/29] DOC: add maxmind geoip disclaimer fixes certtools/intelmq#1110 --- CHANGELOG.md | 3 +++ contrib/cron-jobs/maxmind | 3 +++ intelmq/bots/experts/maxmind_geoip/README.md | 3 +++ intelmq/bots/experts/maxmind_geoip/expert.py | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20ce36327..cf8026c47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ CHANGELOG ### Bots - `collectors.mail.collector_mail_url`: Fix bug which prevented marking emails seen due to disconnects from server (#852). +### Documentation +- Add disclaimer on maxmind database in bot documentation and code and the cron-job (#1110) + 1.0.1 Bugfix release -------------------- ### Documentation diff --git a/contrib/cron-jobs/maxmind b/contrib/cron-jobs/maxmind index 7ed4cf6c8..744455ade 100644 --- a/contrib/cron-jobs/maxmind +++ b/contrib/cron-jobs/maxmind @@ -17,6 +17,9 @@ # 02 01 * * 1 ( cd /tmp; /my/path/to/this/script ) # +# This product includes GeoLite2 data created by MaxMind, available from +# http://www.maxmind.com. + #export http_proxy=http://proxy.cert.at:8080/ path=/usr/local/bin diff --git a/intelmq/bots/experts/maxmind_geoip/README.md b/intelmq/bots/experts/maxmind_geoip/README.md index 207f78171..08faad161 100644 --- a/intelmq/bots/experts/maxmind_geoip/README.md +++ b/intelmq/bots/experts/maxmind_geoip/README.md @@ -8,3 +8,6 @@ ``` "database": "/opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb" ``` + +This product includes GeoLite2 data created by MaxMind, available from +http://www.maxmind.com. diff --git a/intelmq/bots/experts/maxmind_geoip/expert.py b/intelmq/bots/experts/maxmind_geoip/expert.py index f94828c9c..c01b9d6b8 100644 --- a/intelmq/bots/experts/maxmind_geoip/expert.py +++ b/intelmq/bots/experts/maxmind_geoip/expert.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +""" +This product includes GeoLite2 data created by MaxMind, available from +http://www.maxmind.com. +""" from intelmq.lib.bot import Bot From 475db3de9c39b197b83481bd1634f032c768b019 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 6 Nov 2017 11:38:03 +0100 Subject: [PATCH 27/29] REL: Release version 1.0.2 --- CHANGELOG.md | 8 ++++++++ NEWS.md | 4 ++++ debian/changelog | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf8026c47..127dc54bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ CHANGELOG ### Bots - `collectors.mail.collector_mail_url`: Fix bug which prevented marking emails seen due to disconnects from server (#852). +- `parsers.spamhaus.parser_cert`: Handle/ignore 'AS?' in feed (#1111) + +### Packaging +- Support building for more distributions +- Use LSB-paths (/etc/intelmq/, /var/lib/intelmq/, /run/intelmq/) (#470) +- Use quilt instead of native +- Fix problems in postint and postrm scripts +- Use systemd-tmpfile for creation of /run/intelmq/ ### Documentation - Add disclaimer on maxmind database in bot documentation and code and the cron-job (#1110) diff --git a/NEWS.md b/NEWS.md index e338b69b3..62f0e35f6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,10 @@ NEWS See the changelog for a full list of changes. +1.0.2 Bugfix release +-------------------- +No changes needed. + 1.0.1 Bugfix release -------------------- No changes needed. diff --git a/debian/changelog b/debian/changelog index e1a492156..85abb5057 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +intelmq (1.0.2-1) UNRELEASED; urgency=low + + * Update to version 1.0.2 + + -- Wagner Sebastian Mon, 06 Nov 2017 11:36:58 +0100 + intelmq (1.0.1-2) experimental; urgency=low * New base branch 'packaging' for packages From cd2a5ae6e762050920d4f058e7f5db2860abfc45 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 7 Nov 2017 14:07:19 +0100 Subject: [PATCH 28/29] REL: 1.0.2 changelog fixes --- CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 127dc54bb..6bce9a857 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,18 @@ CHANGELOG 1.0.2 Bugfix release -------------------- +### Core +- `lib.message.add`: parameter force has finally been removed, should have been gone in 1.0.0.rc1 already + ### Bots - `collectors.mail.collector_mail_url`: Fix bug which prevented marking emails seen due to disconnects from server (#852). - `parsers.spamhaus.parser_cert`: Handle/ignore 'AS?' in feed (#1111) ### Packaging -- Support building for more distributions -- Use LSB-paths (/etc/intelmq/, /var/lib/intelmq/, /run/intelmq/) (#470) -- Use quilt instead of native +- The following changes have been in effect for the built packages already since version 1.0.0 +- Support building for more distributions, now supported: CentOS 7, Debian 8 and 9, Fedora 25 and 26, RHEL 7, openSUSE Leap 42.2 and 42.3 and Tumbleweed, Ubuntu 14.04 and 16.04 +- Use LSB-paths for created packages (/etc/intelmq/, /var/lib/intelmq/, /run/intelmq/) (#470). Does does not affect installations with setuptools/pip. +- Change the debian package format from native to quilt - Fix problems in postint and postrm scripts - Use systemd-tmpfile for creation of /run/intelmq/ From ddcf76646f7aeaf52d6ed2ce99d13cbce8692847 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 7 Nov 2017 16:44:25 +0100 Subject: [PATCH 29/29] REL: bump version 1.0.2 --- debian/changelog | 2 +- intelmq/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 85abb5057..6c366df9a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -intelmq (1.0.2-1) UNRELEASED; urgency=low +intelmq (1.0.2-1) experimental; urgency=low * Update to version 1.0.2 diff --git a/intelmq/version.py b/intelmq/version.py index a770be9ca..1a6d33da9 100644 --- a/intelmq/version.py +++ b/intelmq/version.py @@ -1,2 +1,2 @@ -__version_info__ = ('1', '0', '1') +__version_info__ = ('1', '0', '2') __version__ = '.'.join(__version_info__)