Merge branch 'maintenance'

.travis.yml

@@ -11,7 +11,7 @@ python:
  - "3.5"
  - "3.6"
 install:
- - sudo apt-get install dpkg-dev dh-python python-setuptools python3-setuptools python3-all debhelper quilt polipo lighttpd fakeroot
+ - sudo apt-get install dpkg-dev dh-python python-setuptools python3-setuptools python3-all debhelper quilt polipo lighttpd fakeroot dh-systemd
  - if [[ $REQUIREMENTS == true ]] ; then for file in intelmq/bots/*/*/REQUIREMENTS.txt; do pip install -r $file; done; fi
  - if [[ $TRAVIS_PYTHON_VERSION < '3.5' ]]; then pip install typing; fi
  - pip install codecov
@@ -26,10 +26,19 @@ before_script:
  - intelmq_psql_initdb
  - sed -i 's/events/tests/g' /tmp/initdb.sql
  - psql -v ON_ERROR_STOP=on -f /tmp/initdb.sql intelmq -U intelmq
+ - VERSION=$(git describe --abbrev=0 --tags)
+ - git archive --format=tar.gz HEAD > ../intelmq_$VERSION.orig.tar.gz
+ - git archive --format=tar.gz --prefix=debian/ HEAD:debian/ > ../intelmq_$VERSION-1.debian.tar.gz
+ - pushd ..
+ - mkdir build
+ - cd build
+ - tar -xzf ../intelmq_$VERSION.orig.tar.gz
+ - tar -xzf ../intelmq_$VERSION-1.debian.tar.gz
+ - popd
 script:
  - if [[ $REQUIREMENTS == true ]] ; then INTELMQ_TEST_DATABASES=1 INTELMQ_TEST_LOCAL_WEB=1 INTELMQ_TEST_EXOTIC=1 nosetests --with-coverage --cover-package=intelmq --cover-branches; else INTELMQ_TEST_LOCAL_WEB=1 nosetests --with-coverage --cover-package=intelmq --cover-branches; fi
- - dpkg-buildpackage -us -uc
  - pycodestyle intelmq/{bots,lib,bin}
+ - pushd ../build && dpkg-buildpackage -us -uc; popd
 services:
  - redis-server
  - postgresql

CHANGELOG.md

@@ -1,6 +1,27 @@
 CHANGELOG
 ==========
 
+1.0.2 Bugfix release
+--------------------
+
+### Core
+- `lib.message.add`: parameter force has finally been removed, should have been gone in 1.0.0.rc1 already
+
+### Bots
+- `collectors.mail.collector_mail_url`: Fix bug which prevented marking emails seen due to disconnects from server (#852).
+- `parsers.spamhaus.parser_cert`: Handle/ignore 'AS?' in feed (#1111)
+
+### Packaging
+- The following changes have been in effect for the built packages already since version 1.0.0
+- Support building for more distributions, now supported: CentOS 7, Debian 8 and 9, Fedora 25 and 26, RHEL 7, openSUSE Leap 42.2 and 42.3 and Tumbleweed, Ubuntu 14.04 and 16.04
+- Use LSB-paths for created packages (/etc/intelmq/, /var/lib/intelmq/, /run/intelmq/) (#470). Does does not affect installations with setuptools/pip.
+- Change the debian package format from native to quilt
+- Fix problems in postint and postrm scripts
+- Use systemd-tmpfile for creation of /run/intelmq/
+
+### Documentation
+- Add disclaimer on maxmind database in bot documentation and code and the cron-job (#1110)
+
 1.0.1 Bugfix release
 --------------------
 ### Documentation
@@ -66,7 +87,7 @@ CHANGELOG
 - `bots.parsers.alienvault.parser_otx`: handle timestamps without floating point seconds
 
 ### Experts
-- bots.experts.deduplicator: New parameter `bypass` to deactivate deduplication, default: true
+- bots.experts.deduplicator: New parameter `bypass` to deactivate deduplication, default: False
 
 v1.0.0.dev8
 -----------

NEWS.md

@@ -3,6 +3,10 @@ NEWS
 
 See the changelog for a full list of changes.
 
+1.0.2 Bugfix release
+--------------------
+No changes needed.
+
 1.0.1 Bugfix release
 --------------------
 No changes needed.

contrib/cron-jobs/maxmind

@@ -17,6 +17,9 @@
 # 02 01 * * 1 ( cd /tmp; /my/path/to/this/script )
 # 
 
+# This product includes GeoLite2 data created by MaxMind, available from
+# <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+
 #export http_proxy=http://proxy.cert.at:8080/
 
 path=/usr/local/bin

debian/changelog

@@ -1,3 +1,15 @@
+intelmq (1.0.2-1) experimental; urgency=low
+
+ * Update to version 1.0.2
+
+ -- Wagner Sebastian <[email protected]> Mon, 06 Nov 2017 11:36:58 +0100
+
+intelmq (1.0.1-2) experimental; urgency=low
+
+ * New base branch 'packaging' for packages
+
+ -- Wagner Sebastian <[email protected]> Wed, 20 Sep 2017 16:26:59 +0200
+
 intelmq (1.0.1-1) experimental; urgency=low
 
  * update to version 1.0.1
@@ -20,6 +32,10 @@ intelmq (1.0.0.rc1-1) experimental; urgency=medium
 
  * update to version 1.0.0.rc1
 
+intelmq (1.0.0.dev8-2) experimental; urgency=medium
+
+ * packaging fixes
+
  -- Sebastian Wagner <[email protected]> Wed, 20 Jun 2017 16:05:00 +0200
 
 intelmq (1.0.0.dev8-1) experimental; urgency=medium

debian/conffiles

@@ -1,5 +1,5 @@
-/opt/intelmq/etc/harmonization.conf
-/opt/intelmq/etc/BOTS
-/opt/intelmq/etc/pipeline.conf
-/opt/intelmq/etc/defaults.conf
-/opt/intelmq/etc/runtime.conf
+/etc/intelmq/harmonization.conf
+/etc/intelmq/BOTS
+/etc/intelmq/pipeline.conf
+/etc/intelmq/defaults.conf
+/etc/intelmq/runtime.conf

debian/control

@@ -2,22 +2,21 @@ Source: intelmq
 Maintainer: Sascha Wilde <[email protected]>
 Section: python
 Priority: optional
-Build-Depends: dh-python, python-setuptools (>= 0.6b3), python3-setuptools, python3-all, debhelper (>= 9), quilt
-X-Python3-Version: >= 3.3.0
+Build-Depends: debhelper (>= 4.1.16), python3-all, python3-setuptools, quilt, dh-python, python-setuptools, dh-systemd, safe-rm, python3-requests, python3-redis, python3-dnspython, python3-psutil, python3-dateutil, python3-termstyle, python3-tz, lsb-release
+X-Python3-Version: >= 3.3
 Standards-Version: 3.9.6
 Homepage: https:/certtools/intelmq/
 
 Package: intelmq
 Architecture: all
 Depends: ${misc:Depends}, ${python3:Depends},
- libffi-dev, python3-dateutil (>= 2.0), python3-dnspython (>= 1.11.1),
- python3-pymongo, python3-openssl, python3-psutil (>= 1.2.1),
- python3-psycopg2, python3-redis (>= 2.10.3),
+ python3-dateutil (>= 2.5), python3-dnspython (>= 1.11.1),
+ python3-openssl, python3-psutil (>= 1.2.1), python3-redis (>= 2.10),
  python3-requests (>= 2.2.1), python3-termstyle (>= 0.1.10), python3-tz,
- redis-server, cron, bash-completion, jq
-Recommends: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0),
+ redis-server, cron, bash-completion, jq, systemd
+Suggests: python3-imbox (>= 0.8), python3-pyasn (>= 1.5.0),
  python3-stomp.py (>= 4.1.9), python3-sleekxmpp (>= 1.3.1),
- python3-geoip2 (>= 2.2.0)
+ python3-geoip2 (>= 2.2.0), python3-pymongo, python3-psycopg2
 Description: IntelMQ is a solution for IT security teams (CERTs, CSIRTs, abuse
  departments,...) for collecting and processing security feeds (such as log
  files) using a message queuing protocol. It's a community driven initiative

debian/cron.d/intelmq-update-data

@@ -3,10 +3,10 @@
 #
 # m h dom mon dow command
 # Update data for tor_nodes bot:
-11 0 * * * intelmq /usr/bin/update-tor-nodes /opt/intelmq/var/lib/bots/tor_nodes/tor_nodes.dat
+11 0 * * * intelmq /usr/bin/update-tor-nodes /var/lib/intelmq/bots/tor_nodes/tor_nodes.dat
 # Update data for maxmind_geoip bot:
-17 0 * * * intelmq /usr/bin/update-geoip-data /opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb
+17 0 * * * intelmq /usr/bin/update-geoip-data /var/lib/intelmq/bots/maxmind_geoip/GeoLite2-City.mmdb
 # Update data for asn_lookup bot:
-23 0 * * * intelmq /usr/bin/update-asn-data /opt/intelmq/var/lib/bots/asn_lookup/ipasn.dat
+23 0 * * * intelmq /usr/bin/update-asn-data /var/lib/intelmq/bots/asn_lookup/ipasn.dat
 # Update data for the RIPE DB abuse_c offline contact lookup
-25 6 * * * intelmq /usr/bin/update-ripencc_abuse_contact_offline /opt/intelmq/var/lib/bots/ripencc_abuse_contact_offline/
+25 6 * * * intelmq /usr/bin/update-ripencc_abuse_contact_offline /var/lib/intelmq/bots/ripencc_abuse_contact_offline/

debian/intelmq.postinst

@@ -2,11 +2,15 @@
 set -e
 
 if [ "$1" = "configure" ] ; then
+ if ! getent group intelmq >/dev/null 2>&1; then
+ groupadd -r intelmq
+ fi
  if ! getent passwd intelmq >/dev/null 2>&1; then
- useradd -d /opt/intelmq -U -s /bin/bash intelmq
+ useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
  fi
- chmod -R g+w /opt/intelmq/*
- chown -R intelmq:intelmq /opt/intelmq
+ chmod -R g+w /var/lib/intelmq/*
+ chown -R intelmq:intelmq /etc/intelmq/ /var/lib/intelmq/ /var/log/intelmq/
+ systemd-tmpfiles --create /usr/lib/tmpfiles.d/intelmq.conf
 fi
 
 #DEBHELPER#

debian/intelmq.postrm

@@ -2,7 +2,12 @@
 set -e
 
 if [ "$1" = "remove" ] ; then
- deluser intelmq
+ if getent group intelmq >/dev/null 2>&1; then
+ groupdel -r intelmq
+ fi
+ if getent passwd intelmq >/dev/null 2>&1; then
+ userdel intelmq
+ fi
 fi
 
 #DEBHELPER#

debian/intelmq.tmpfile

@@ -0,0 +1 @@
+D /run/intelmq 0755 intelmq intelmq

debian/patches/fix-dnspython-name.patch

@@ -2,7 +2,7 @@ Description: Undo rename of the dependency dnspython3
  The packages was renamed upstream; however, this change does not immediately
  affect this package's target distributions.
 Author: Gernot Schulz <[email protected]>
-Last-Update: 2016-07-20
+Last-Update: 2016-10-04
 ---
 This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
 --- a/setup.py

debian/patches/fix-intelmq-paths.patch

@@ -0,0 +1,24 @@
+Description: Set all paths to LSB
+Author: Sebastian Wagner <[email protected]>
+Last-Update: 2017-09-20
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/intelmq/__init__.py
++++ b/intelmq/__init__.py
+@@ -1,13 +1,12 @@
+ from .version import __version__
+ import os
+
+-ROOT_DIR = "/opt/intelmq/"
+-CONFIG_DIR = os.path.join(ROOT_DIR, "etc/")
++CONFIG_DIR = "/etc/intelmq/"
+ DEFAULT_LOGGING_LEVEL = "INFO"
+ BOTS_FILE = os.path.join(CONFIG_DIR, "BOTS")
+-DEFAULT_LOGGING_PATH = os.path.join(ROOT_DIR, "var/log/")
++DEFAULT_LOGGING_PATH = "/var/log/intelmq/"
+ DEFAULTS_CONF_FILE = os.path.join(CONFIG_DIR, "defaults.conf")
+ HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf")
+ PIPELINE_CONF_FILE = os.path.join(CONFIG_DIR, "pipeline.conf")
+ RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.conf")
+-VAR_RUN_PATH = os.path.join(ROOT_DIR, "var/run/")
++VAR_RUN_PATH = "/var/run/intelmq/"

debian/patches/series

@@ -1,2 +1,2 @@
 fix-logrotate-path.patch
-fix-dnspython-name.patch
+fix-intelmq-paths.patch

debian/rules

@@ -1,34 +1,60 @@
 #!/usr/bin/make -f
 
+CODENAME := $(shell lsb_release -cs)
 DOCS_DIR := debian/intelmq/usr/share/doc/intelmq
+export INTELMQ_SKIP_REDIS=1
+export INTELMQ_SKIP_INTERNET=1
 
 # Compile list of bot READMEs
 BOTROOT := intelmq/bots
 BOTCATEGORIES := collectors experts outputs parsers
 BOTCATEGORIES := $(foreach cat,$(BOTCATEGORIES),$(BOTROOT)/$(cat))
-BOTS := $(foreach bot,$(BOTCATEGORIES),$(wildcard $(bot)/*))
+BOTS := $(foreach bot,$(BOTCATEGORIES),$(wildcard $(bot)/*))
 BOTDOCS := $(foreach bot,$(BOTS),$(wildcard $(bot)/*.md))
 
 # This file is based on a version
 # automatically generated by stdeb 0.8.5 at
 # Wed, 23 Mar 2016 17:49:26 +0000
 export PYBUILD_NAME=intelmq
 %:
- dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt
+ dh $@ --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd
+
+build:
+ if [ $(CODENAME) = 'xenial' ] || [ $(CODENAME) = 'jessie' ]; then\
+ patch -p1 setup.py debian/patches/fix-dnspython-name.patch;\
+ fi
+ dh build --with python3 --without python2 --buildsystem=pybuild --with quilt --with systemd
 
 override_dh_auto_build:
 
 override_dh_auto_install: $(BOTDOCS)
+ rm intelmq/bin/rewrite_config_files.py
+ rm intelmq/bin/intelmq_gen_harm_docs.py
+ sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py
+ find . -type f -exec sed -i -f debian/sedfile {} \;
  python3 setup.py install --root=debian/intelmq --prefix=/usr
- mv debian/intelmq/opt/intelmq/etc/examples/* debian/intelmq/opt/intelmq/etc/
- rmdir debian/intelmq/opt/intelmq/etc/examples
- mkdir -p debian/intelmq/opt/intelmq/var/log
- mkdir -p debian/intelmq/opt/intelmq/var/lib/bots/file-output
+ mv debian/intelmq/etc/intelmq/examples/* debian/intelmq/etc/intelmq/
+ rmdir debian/intelmq/etc/intelmq/examples
+ # these are already in /usr/bin/
+ #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/maxmind_geoip/update-geoip-data
+ #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/asn_lookup/update-asn-data
+ #rm %{buildroot}/%{python3_sitelib}/intelmq/bots/experts/tor_nodes/update-tor-nodes
+ # and rename those in /usr/bin
+ mv debian/intelmq/usr/bin/update-geoip-data debian/intelmq/usr/bin/intelmq-update-geoip-data
+ mv debian/intelmq/usr/bin/update-asn-data debian/intelmq/usr/bin/intelmq-update-asn-data
+ mv debian/intelmq/usr/bin/update-tor-nodes debian/intelmq/usr/bin/intelmq-update-tor-nodes
+ # create directories
+ mkdir -p debian/intelmq/var/log/intelmq
+ mkdir -p debian/intelmq/var/lib/intelmq/bots/file-output
  mkdir -p debian/intelmq/etc/logrotate.d
+ mkdir -p debian/intelmq/var/lib/intelmq/bots/modify/
  ## BOTS
  # Include all bot READMEs
  for readme in $(foreach bot,$(BOTDOCS),$(subst intelmq/bots/,,$(bot))); \
  do \
  mkdir -p $(DOCS_DIR)/bots/$$(dirname $$readme); \
  cp intelmq/bots/$$readme $(DOCS_DIR)/bots/$$(dirname $$readme); \
  done
+
+override_dh_systemd_start:
+ dh_systemd_start --restart-after-upgrade

debian/sedfile

@@ -0,0 +1,5 @@
+s/opt\/intelmq\/var\/run/var\/run\/intelmq/g
+s/opt\/intelmq\/var\/log/var\/log\/intelmq/g
+s/opt\/intelmq\/var\/lib/var\/lib\/intelmq/g
+s/opt\/intelmq\/etc\//etc\/intelmq\//g
+s/opt\/intelmq/etc\/intelmq/g

debian/source/format

@@ -1 +1 @@
-3.0 (native)
+3.0 (quilt)

docs/INSTALL.md

@@ -111,11 +111,11 @@ https://software.opensuse.org/download.html?project=home%3Asebix%3Aintelmq&packa
 
 Currently, these operating systems are supported by the packages:
 * CentOS 7, install `epel-release` first
-* Debian 8, install `python3-typing` too
-* Fedora 25
-* openSUSE Leap 42.2 and 42.3
-* openSUSE Tumbleweed
-* Ubuntu 16.04
+* RHEL 7, install `epel-release` first
+* Debian 8 (install `python3-typing` too) and 9
+* Fedora 25, 26 and Rawhide
+* openSUSE Leap 42.2, 42.3 and Tumbleweed
+* Ubuntu 16.04 and 17.04
 
 Please report any errors or improvements at https:/certtools/intelmq/issues Thanks!
 
@@ -158,6 +158,7 @@ sudo -s
 
 pip3 install .
 
+mkdir /opt/intelmq
 useradd -d /opt/intelmq -U -s /bin/bash intelmq
 chmod -R 0770 /opt/intelmq
 chown -R intelmq.intelmq /opt/intelmq

intelmq/bots/collectors/mail/collector_mail_url.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import re
 import io
+import imaplib
 import requests
 
 try:
@@ -26,11 +27,15 @@ def init(self):
  self.chunk_replicate_header = getattr(self.parameters,
  'chunk_replicate_header', None)
 
- def process(self):
+ def connect_mailbox(self):
  mailbox = imbox.Imbox(self.parameters.mail_host,
  self.parameters.mail_user,
  self.parameters.mail_password,
  self.parameters.mail_ssl)
+ return mailbox
+
+ def process(self):
+ mailbox = self.connect_mailbox()
  emails = mailbox.messages(folder=self.parameters.folder, unread=True)
 
  if emails:
@@ -90,7 +95,12 @@ def process(self):
  # Only mark read if message relevant to this instance,
  # so other instances watching this mailbox will still
  # check it.
- mailbox.mark_seen(uid)
+ try:
+ mailbox.mark_seen(uid)
+ except imaplib.abort:
+ # Disconnect, see https:/certtools/intelmq/issues/852
+ mailbox = self.connect_mailbox()
+ mailbox.mark_seen(uid)
 
  if not erroneous:
  self.logger.info("Email report read.")

intelmq/bots/experts/maxmind_geoip/README.md

@@ -8,3 +8,6 @@
 ```
  "database": "/opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb"
 ```
+
+This product includes GeoLite2 data created by MaxMind, available from
+<a href="http://www.maxmind.com">http://www.maxmind.com</a>.

intelmq/bots/experts/maxmind_geoip/expert.py

@@ -1,4 +1,8 @@
 # -*- coding: utf-8 -*-
+"""
+This product includes GeoLite2 data created by MaxMind, available from
+<a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+"""
 
 from intelmq.lib.bot import Bot