[mimecast] Add use cases, docs, and update sample events

packages/mimecast/changelog.yml

@@ -1,3 +1,8 @@
+- version: "0.0.6"
+ changes:
+ - description: Add use cases for audit events and update sample events and docs
+ type: bugfix
+ link: https:/elastic/integrations/pull/2690
 - version: "0.0.5"
  changes:
  - description: Fix typo

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log

@@ -22,4 +22,6 @@
 {"id":"eNqrVipOTS4tSs1MUbJSigzJC_ZNzg-vcjYKcwz3icotC0nVdgtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYG5kqaOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAD-SK84","auditType":"Archive Mailbox Restore","user":"[email protected]","eventTime":"2021-10-12T18:19:33+0000","eventInfo":"Archive mailbox restore created. Restored data from [email protected] to [email protected] by [email protected], Date: 2021-10-12, Time: 18:19:33+0000, IP: 67.43.156.15, Application: Administration Console","category":"archive_service_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJScjMvyjIxr6yoLDY2qQopLq3yDnM1dwtL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxiYGZorKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE5dK-0","auditType":"Archive Mailbox Export Download","user":"[email protected]","eventTime":"2021-10-12T17:55:14+0000","eventInfo":"Mailbox export downloaded. Download filename (HTML Report recovery id): eNqrVipOTS4tSs1MUbJSyo3RDw81rTCpynMpdiuICMopyihxynZztcisDMoN9zWLSCrPzAjz9PALNzFwySrLMNQ2yUs38g9zS860cHKNMExR0lFKLi0uyc9NLUrOT0kFGulsYuZobgoUL0pNzi9LLarULUksztYFWWdpaKqjBBQqzszPU7IyrAUAsSEteA by [email protected], Date: 2021-10-12, Time: 17:55:14+0000, IP: 67.43.156.15, Application: Administration Console","category":"archive_service_logs"}
 {"id":"eNqrVipOTS4tSs1MUbJSitH39gl1cS509PT1MSnw90l0CinPCQgLS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsaGlsYmBsYqqjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgAxASul","auditType":"Review Set Action","user":"[email protected]","eventTime":"2021-10-12T17:07:00+0000","eventInfo":"Viewed Review Set Details - Case: Class Action, Review Set: Contracts, Date: 2021-10-12, Time: 17:07:00+0000, IP: 67.43.156.15, Application: mimecast-case-review","category":"case_review_logs"}
-{"id":"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38","auditType":"Remediation Incident Adjustment","user":"[email protected]","eventTime":"2021-10-12T15:38:05+0000","eventInfo":"Restore Remediation Incident Created - TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console","category":"account_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJS8vDNLCt0DHEKS4xICvNJqzQ1MjOyyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaWJurKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAByMK38","auditType":"Remediation Incident Adjustment","user":"[email protected]","eventTime":"2021-10-12T15:38:05+0000","eventInfo":"Restore Remediation Incident Created - TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console","category":"account_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"[email protected]","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for [email protected] <John Doe>, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked","category":"authentication_logs"}
+{"id":"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg","auditType":"Logon Authentication Failed","user":"[email protected]","eventTime":"2021-10-12T08:47:55+0000","eventInfo":"Failed authentication for [email protected] <John Doe>, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password","category":"authentication_logs"}

packages/mimecast/data_stream/audit_events/_dev/test/pipeline/test-audit-events.log-expected.json

@@ -1233,6 +1233,106 @@
  "category": "account_logs",
  "eventInfo": "Restore Remediation Incident Created - TR-C46A75-01419-R, type : restore, search criteria : {\"unremediateCode\":\"TR-C46A75-01419-M\",\"from\":\"gmail.com\",\"start\":\"2021-10-10T15:33:49+0000\",\"end\":\"2021-10-12T15:33:49+0000\"}, Date: 2021-10-12, Time: 15:38:05+0000, IP: 67.43.156.15, Application: Administration Console"
  }
- }
+ },
+ {
+ "@timestamp": "2021-10-12T08:47:55.000Z",
+ "client": {
+ "as": {
+ "number": 35908
+ },
+ "geo": {
+ "continent_name": "Asia",
+ "country_iso_code": "BT",
+ "country_name": "Bhutan",
+ "location": {
+ "lat": 27.5,
+ "lon": 90.5
+ }
+ },
+ "ip": "67.43.156.15"
+ },
+ "ecs": {
+ "version": "1.12.0"
+ },
+ "event": {
+ "action": "logon-authentication-failed",
+ "created": "2022-01-11T22:54:04.000Z",
+ "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg",
+ "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"[email protected]\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for [email protected] \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked\",\"category\":\"authentication_logs\"}",
+ "reason": "Account Locked"
+ },
+ "mimecast": {
+ "application": "POP-POP2",
+ "category": "authentication_logs",
+ "eventInfo": "Failed authentication for [email protected] \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 22:54:04 GMT, IP: 67.43.156.15, Application: POP-POP2, Reason: Account Locked"
+ },
+ "related": {
+ "ip": [
+ "67.43.156.15"
+ ],
+ "user": [
+ "johndoe",
+ "[email protected]"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "email": "[email protected]",
+ "name": "johndoe"
+ }
+ },
+ {
+ "@timestamp": "2021-10-12T08:47:55.000Z",
+ "client": {
+ "as": {
+ "number": 35908
+ },
+ "geo": {
+ "continent_name": "Asia",
+ "country_iso_code": "BT",
+ "country_name": "Bhutan",
+ "location": {
+ "lat": 27.5,
+ "lon": 90.5
+ }
+ },
+ "ip": "67.43.156.15"
+ },
+ "ecs": {
+ "version": "1.12.0"
+ },
+ "event": {
+ "action": "logon-authentication-failed",
+ "created": "2022-01-11T21:48:01.000Z",
+ "id": "eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg",
+ "original": "{\"id\":\"eNqrVipOTS4tSs1MUbJSMvCrMHX2MzL1yLFITjJNd8rO9wiJyAlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWxsaGRkoKOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAPktKzg\",\"auditType\":\"Logon Authentication Failed\",\"user\":\"[email protected]\",\"eventTime\":\"2021-10-12T08:47:55+0000\",\"eventInfo\":\"Failed authentication for [email protected] \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password\",\"category\":\"authentication_logs\"}",
+ "reason": "Reason: Wrong Password"
+ },
+ "mimecast": {
+ "application": "POP-POP2",
+ "category": "authentication_logs",
+ "eventInfo": "Failed authentication for [email protected] \u003cJohn Doe\u003e, Date: 2022-01-11, Time: 21:48:01 GMT, IP: 67.43.156.15, Application: POP-POP2, Method: Cloud, Reason: Wrong Password"
+ },
+ "related": {
+ "ip": [
+ "67.43.156.15"
+ ],
+ "user": [
+ "johndoe",
+ "[email protected]"
+ ]
+ },
+ "tags": [
+ "preserve_original_event"
+ ],
+ "user": {
+ "domain": "example.com",
+ "email": "[email protected]",
+ "name": "johndoe"
+ }
+ }
  ]
 }

packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml

@@ -22,7 +22,15 @@ processors:
  field: mimecast.eventTime
  timezone: UTC
  formats:
- - yyyy-MM-dd'T'HH:mm:ssZ
+ - "yyyy-MM-dd'T'HH:mm:ssz"
+ - "yyyy-MM-dd'T'HH:mm:ssZ"
+ - "yyyy-MM-dd'T'HH:mm:ss.Sz"
+ - "yyyy-MM-dd'T'HH:mm:ss.SZ"
+ - "yyyy-MM-dd'T'HH:mm:ss.SSz"
+ - "yyyy-MM-dd'T'HH:mm:ss.SSZ"
+ - "yyyy-MM-dd'T'HH:mm:ss.SSSz"
+ - "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
+ - "yyyy-MM-dd'T'HH:mm:ss z"
 
  ###
 
@@ -93,7 +101,18 @@ processors:
  - dissect:
  field: mimecast.eventInfo
  pattern: "%{mimecast.info}, %{?key}:%{mimecast.email.address}[%{mimecast.email.metadata}] %{?key}: %{client.ip} %{?key}: %{mimecast.application}"
- if: 'ctx?.event?.action=="logon-authentication-failed"'
+ if: 'ctx?.event?.action=="logon-authentication-failed" && (ctx?.mimecast?.email?.metadata != "")'
+ ignore_missing: true
+ ignore_failure: true
+ - split:
+ field: mimecast.eventInfo
+ separator: ","
+ target_field: mimecast.event_info_parts
+ if: 'ctx?.mimecast?.eventInfo != null && ctx?.event?.action=="logon-authentication-failed"'
+ - dissect:
+ field: mimecast.eventInfo
+ pattern: "%{mimecast.info}, %{?key}: %{mimecast.date}, %{?key}: %{mimecast.time}, %{?key}: %{client.ip}, %{?key}: %{mimecast.application}, %{?key}: %{event.reason}"
+ if: 'ctx?.event?.action=="logon-authentication-failed" && (ctx?.mimecast?.event_info_parts.length == 6)'
  ignore_missing: true
  ignore_failure: true
  - dissect:
@@ -152,6 +171,15 @@ processors:
  - yyyy-MM-dd HH:mm:ssZ
  - yyyy-MM-dd HH:mm:ss z
  - yyyy-MM-dd HH:mm:ss
+ - yyyy-MM-dd'T'HH:mm:ssz
+ - yyyy-MM-dd'T'HH:mm:ssZ
+ - yyyy-MM-dd'T'HH:mm:ss.Sz
+ - yyyy-MM-dd'T'HH:mm:ss.SZ
+ - yyyy-MM-dd'T'HH:mm:ss.SSz
+ - yyyy-MM-dd'T'HH:mm:ss.SSZ
+ - yyyy-MM-dd'T'HH:mm:ss.SSSz
+ - yyyy-MM-dd'T'HH:mm:ss.SSSZ
+ - yyyy-MM-dd'T'HH:mm:ss z
  if: 'ctx?.event?.created != null'
  - geoip:
  field: client.ip
@@ -221,6 +249,7 @@ processors:
  - mimecast.columns_exported
  - mimecast.as.asn
  - mimecast.organization_name
+ - mimecast.event_info_parts
  ignore_missing: true
  - remove:
  description: Remove 'event.original' if 'preserve_original_event' is not set.

packages/mimecast/data_stream/audit_events/sample_event.json

@@ -1,12 +1,25 @@
 {
- "@timestamp": "2021-11-16T12:01:37.000Z",
- "agent": {
- "ephemeral_id": "57841034-22ed-4fcd-bcfd-0a9518249e2d",
- "hostname": "docker-fleet-agent",
- "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab",
- "name": "docker-fleet-agent",
- "type": "filebeat",
- "version": "7.16.0"
+ "@timestamp": "2022-02-09T02:45:01.000Z",
+ "file": {
+ "extension": "zip",
+ "name": "Threat intel multiple feeds download - malware_customer_csv_20220209024500934.zip"
+ },
+ "ecs": {
+ "version": "1.12.0"
+ },
+ "related": {
+ "ip": [
+ "8.8.8.8"
+ ],
+ "user": [
+ "johndoe",
+ "[email protected]"
+ ]
+ },
+ "data_stream": {
+ "namespace": "default",
+ "type": "logs",
+ "dataset": "mimecast.audit_events"
  },
  "client": {
  "as": {
@@ -26,53 +39,22 @@
  },
  "ip": "8.8.8.8"
  },
- "data_stream": {
- "dataset": "mimecast.audit_events",
- "namespace": "ep",
- "type": "logs"
- },
- "ecs": {
- "version": "1.12.0"
- },
- "elastic_agent": {
- "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab",
- "snapshot": true,
- "version": "7.16.0"
- },
  "event": {
- "action": "case-action",
  "agent_id_status": "verified",
- "created": "2021-11-16T12:01:37.000Z",
- "dataset": "mimecast.audit_events",
- "id": "eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI",
- "ingested": "2021-11-24T15:39:11Z",
- "original": "{\"auditType\":\"Case Action\",\"category\":\"case_review_logs\",\"eventInfo\":\"Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review\",\"eventTime\":\"2021-11-16T12:01:37+0000\",\"id\":\"eNqrVipOTS4tSs1MUbJSskwzjDIMyDRKLinNSEl1c0pOqXLJyQlL89c2DXZ1C3eP9AyvijKL9I7Rd_WOzC0ztMg2dzFM1M73s6w09CqoDA1T0lFKLE3JLMnJTwcZaGxoaWFsYmhkrqOUXFpckp-bWpScn5IKtMnZxMzR3BSovCy1qDgzP0_JyrAWAE_sLAI\",\"user\":\"[email protected]\"}"
+ "ingested": "2022-02-09T09:45:25Z",
+ "created": "2022-02-09T02:45:01.000Z",
+ "action": "threat-intel-feed-download",
+ "id": "eNqrVipOTS4tSs1MUbJSyvMxyknzzcqN0S9Nzs_PqCoNCTE2j3ILS_PXNg12dQt3j_QMr4oyi_SO0Xf1jswtM7TINncxTNTO97OsNPQqqAwNU9JRSixNySzJyU8HmWhsZGhobmJkYKKjlFxaXJKfm1qUnJ-SCrTK2cTM0dwUqLwstag4Mz9PycqwFgCY1Sx4",
+ "dataset": "mimecast.audit_events"
  },
- "input": {
- "type": "httpjson"
- },
- "mimecast": {
- "application": "mimecast-case-review",
- "category": "case_review_logs",
- "eventInfo": "Viewed Case - Case: GDPR/CCPA, Date: 2021-11-16, Time: 12:01:37+0000, IP: 8.8.8.8, Application: mimecast-case-review"
- },
- "related": {
- "ip": [
- "8.8.8.8"
- ],
- "user": [
- "johndoe",
- "[email protected]"
- ]
- },
- "tags": [
- "preserve_original_event",
- "forwarded",
- "mimecast-audit-events"
- ],
  "user": {
  "domain": "example.com",
- "email": "[email protected]",
- "name": "johndoe"
+ "name": "johdoe",
+ "email": "[email protected]"
+ },
+ "mimecast": {
+ "eventInfo": "Threat intel multiple feeds download - malware_customer_csv_20220209024500934.zip, Date: 2022-02-09, Time: 02:45:01+0000, IP: 8.8.8.8, Application: Integrations",
+ "application": "Integrations",
+ "category": "reporting_logs"
  }
 }

packages/mimecast/data_stream/dlp_logs/sample_event.json

@@ -1,13 +1,5 @@
 {
  "@timestamp": "2021-11-18T21:41:18.000Z",
- "agent": {
- "ephemeral_id": "1aef981f-3448-4d12-bd5a-723ac1cdcc81",
- "hostname": "docker-fleet-agent",
- "id": "eb7f38a3-c00c-4d87-9c69-fddb3d650fab",
- "name": "docker-fleet-agent",
- "type": "filebeat",
- "version": "7.16.0"
- },
  "data_stream": {
  "dataset": "mimecast.dlp_logs",
  "namespace": "ep",

packages/mimecast/data_stream/siem_logs/sample_event.json

@@ -1,33 +1,35 @@
 {
- "@timestamp": "2021-10-18T08:02:43.000Z",
+ "@timestamp": "2022-02-03T18:17:38.000Z",
  "ecs": {
  "version": "1.12.0"
  },
+ "data_stream": {
+ "namespace": "default",
+ "type": "logs",
+ "dataset": "mimecast.siem_logs"
+ },
  "event": {
- "reason": "Spm",
- "action": "Hld",
- "ingested": "2021-11-25T11:34:11.459620200Z",
- "original": "{\"Act\":\"Hld\",\"AttCnt\":0,\"AttNames\":null,\"AttSize\":0,\"Content-Disposition\":\"attachment; filename=\\\"process_20211018093329655.json\\\"\",\"Hld\":\"Spm\",\"MsgId\":\"\\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\\u003e\",\"MsgSize\":157436,\"Sender\":\"bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu\",\"Subject\":\"Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!\",\"aCode\":\"HhuwRf_AOcuJZINE2ZgcKw\",\"acc\":\"ABC123\",\"datetime\":\"2021-10-18T09:02:43+0100\"}",
- "created": "2021-10-18T09:02:43+0100",
+ "agent_id_status": "verified",
+ "ingested": "2022-02-09T09:58:25Z",
+ "created": "2022-02-03T18:17:38+0000",
+ "action": "Acc",
+ "dataset": "mimecast.siem_logs",
  "outcome": "unknown"
  },
  "email": {
- "message_id": "\u003cINX.164dae0719be95da77068c7d264.3e915.e7719.c78c.17c926a3231ace@newsletter.77onlineshop.eu\u003e",
- "from": {
- "address": "bounce_9244+cdaahhimyaaaaagaad5ekqaaaaaaaaeribenpq@newsletter.77onlineshop.eu"
- },
  "attachments": {
  "file": {
  "size": 0
  }
  },
- "local_id": "HhuwRf_AOcuJZINE2ZgcKw",
- "subject": "Hi Sandra! Neue Styles eingetroffen! – Finde deinen Lieblings-Look!",
- "message_size": 157436
+ "local_id": "23e26c29-14fa-4a31-a6a1-474ba8fa7943",
+ "subject": "You've been sent a secure message: hello world",
+ "message_id": "\[email protected]\u003e",
+ "from": {
+ "address": "[email protected]"
+ },
+ "message_size": 27677
  },
- "tags": [
- "preserve_original_event"
- ],
  "mimecast": {
  "acc": "ABC123",
  "log_type": "process",