HARM: Drop malware.hash, new malware.hash.sha256

fixes #732

Signed-off-by: Sebastian Wagner <[email protected]>

CHANGELOG.md

@@ -100,7 +100,7 @@ v1.0 (in development, master branch)
 - `additional_information` renamed to `extra`, must be JSON
 - `os.name`, `os.version`, `user_agent` removed in favor of `extra`
 - all hashes are lower case only
-- added `malware.hash.(md5|sha1)`
+- added `malware.hash.(md5|sha1|sha256)`, removed `malware.hash`
 - New parameter and field named feed.accuracy to represent the accuracy of each feed
 - New parameter and field named feed.provider to document the name of the source of each feed
 - New parameter and field named feed.documentation to link to documentation of the feed

NEWS.md

@@ -24,8 +24,9 @@ ALTER TABLE events
  ADD COLUMN "feed.accuracy" text,
  ADD COLUMN "feed.documentation" text,
  ADD COLUMN "feed.provider" text,
- ADD COLUMN "malware.hash.md5" text,
- ADD COLUMN "malware.hash.sha1" text,
+ ADD COLUMN "malware.hash.md5" varchar(200),
+ ADD COLUMN "malware.hash.sha1" varchar(200),
+ ADD COLUMN "malware.hash.sha256" varchar(200),
  ADD COLUMN "protocol.transport" text,
  RENAME COLUMN "additional_information" TO "extra",
  RENAME COLUMN "destination.bgp_prefix" TO "destination.network" text,
@@ -62,9 +63,15 @@ UPDATE events
 UPDATE events
  SET "event_hash" = lower("event_hash")
  WHERE "event_hash" IS NOT NULL;
-UPDATE events
- SET "malware.hash" = lower("malware.hash")
- WHERE "malware.hash" IS NOT NULL;
+UPDATE tests
+ SET "malware.hash.md5" = substring("malware.hash" from 4)
+ WHERE substring("malware.hash" from 1 for 3) = '$1$';
+UPDATE tests
+ SET "malware.hash.sha1" = substring("malware.hash" from 7)
+ WHERE substring("malware.hash" from 1 for 6) = '$sha1$';
+UPDATE tests
+ SET "malware.hash.sha256" = substring("malware.hash" from 4)
+ WHERE substring("malware.hash" from 1 for 3) = '$5$';
 UPDATE events
  SET "malware.hash.md5" = lower("malware.hash.md5")
  WHERE "malware.hash.md5" IS NOT NULL;
@@ -75,5 +82,6 @@ UPDATE events
 ALTER TABLE events
  DROP COLUMN "os.name",
  DROP COLUMN "os.version",
- DROP COLUMN "user_agent";
+ DROP COLUMN "user_agent",
+ DROP COLUMN "malware.hash";
 ```

docs/Harmonization-fields.md

@@ -41,9 +41,9 @@ Harmonization field names
 |Feed|feed.name|[String](#string)|Name for the feed, usually found in collector bot configuration.|
 |Feed|feed.provider|[String](#string)|Name for the provider of the feed, usually found in collector bot configuration.|
 |Feed|feed.url|[URL](#url)|The URL of a given abuse feed, where applicable|
-|Malware|malware.hash|[String](#string)|A string depicting a checksum for a file, be it a malware sample for example. You may include the hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29 and use only printable characters. Please see https:/certtools/intelmq/pull/634 for a discussion on this issue.|
-|Malware Hash|malware.hash.md5|[String](#string)|A string depicting an MD5 checksum for a file, be it a malware sample for example. You may include the hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29 and use only printable characters. Please see https:/certtools/intelmq/pull/634 for a discussion on this issue.|
-|Malware Hash|malware.hash.sha1|[String](#string)|A string depicting a SHA1 checksum for a file, be it a malware sample for example. You may include the hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29 and use only printable characters. Please see https:/certtools/intelmq/pull/634 for a discussion on this issue.|
+|Malware Hash|malware.hash.md5|[String](#string)|A string depicting an MD5 checksum for a file, be it a malware sample for example.|
+|Malware Hash|malware.hash.sha1|[String](#string)|A string depicting a SHA1 checksum for a file, be it a malware sample for example.|
+|Malware Hash|malware.hash.sha256|[String](#string)|A string depicting a SHA256 checksum for a file, be it a malware sample for example.|
 |Malware|malware.name|[LowercaseString](#lowercasestring)|A malware family name in lower case.|
 |Malware|malware.version|[String](#string)|A version string for an identified artifact generation, e.g. a crime-ware kit.|
 |Misp|misp.attribute_uuid|[LowercaseString](#lowercasestring)|MISP - Malware Information Sharing Platform & Threat Sharing UUID of an attribute.|

intelmq/bots/parsers/alienvault/parser_otx.py

@@ -12,9 +12,9 @@
 from intelmq.lib.bot import Bot
 
 HASHES = {
- 'FileHash-SHA256': '$5$',
- 'FileHash-SHA1': '$sha1$',
- 'FileHash-MD5': '$1$'
+ 'FileHash-SHA256': 'malware.hash.sha265',
+ 'FileHash-SHA1': 'malware.hash.sha1',
+ 'FileHash-MD5': 'malware.hash.md5'
 }
 
 
@@ -31,8 +31,7 @@ def process(self):
  event = self.new_event(report)
  # hashes
  if indicator["type"] in HASHES.keys():
- event.add('malware.hash', HASHES[indicator["type"]] +
- indicator["indicator"])
+ event.add(HASHES[indicator["type"]], indicator["indicator"])
  # fqdn
  if indicator["type"] in ['hostname', 'domain']:
  event.add('source.fqdn',

intelmq/bots/parsers/n6/parser_n6stomp.py

@@ -88,7 +88,7 @@ def process(self):
  if "dport" in dict_report:
  event.add("destination.port", dict_report["dport"])
  if "md5" in dict_report:
- event.add("malware.hash", dict_report["md5"])
+ event.add("malware.hash.md5", dict_report["md5"])
  if "sha1" in dict_report:
  event.add("malware.hash.sha1", dict_report["sha1"])
  if "fqdn" in dict_report:

intelmq/etc/harmonization.conf

@@ -157,20 +157,20 @@
  "description": "The URL of a given abuse feed, where applicable",
  "type": "URL"
  },
- "malware.hash": {
- "description": "A string depicting a checksum for a file, be it a malware sample for example. You may include the hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29 and use only printable characters. Please see https:/certtools/intelmq/pull/634 for a discussion on this issue.",
+ "malware.hash.md5": {
+ "description": "A string depicting an MD5 checksum for a file, be it a malware sample for example.",
  "length": 200,
  "regex": "^[ -~]+$",
  "type": "String"
  },
- "malware.hash.md5": {
- "description": "A string depicting an MD5 checksum for a file, be it a malware sample for example. You may include the hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29 and use only printable characters. Please see https:/certtools/intelmq/pull/634 for a discussion on this issue.",
+ "malware.hash.sha1": {
+ "description": "A string depicting a SHA1 checksum for a file, be it a malware sample for example.",
  "length": 200,
  "regex": "^[ -~]+$",
  "type": "String"
  },
- "malware.hash.sha1": {
- "description": "A string depicting a SHA1 checksum for a file, be it a malware sample for example. You may include the hash type according to https://en.wikipedia.org/wiki/Crypt_%28C%29 and use only printable characters. Please see https:/certtools/intelmq/pull/634 for a discussion on this issue.",
+ "malware.hash.sha256": {
+ "description": "A string depicting a SHA256 checksum for a file, be it a malware sample for example.",
  "length": 200,
  "regex": "^[ -~]+$",
  "type": "String"

intelmq/tests/bin/initdb.sql

@@ -37,9 +37,9 @@ CREATE TABLE events (
  "feed.name" text,
  "feed.provider" text,
  "feed.url" text,
- "malware.hash" varchar(200),
  "malware.hash.md5" varchar(200),
  "malware.hash.sha1" varchar(200),
+ "malware.hash.sha256" varchar(200),
  "malware.name" text,
  "malware.version" text,
  "misp.attribute_uuid" varchar(36),