[Filebeat] Add module for Kibana audit logs

CHANGELOG.next.asciidoc

@@ -43,6 +43,8 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696]
+
 
 *Auditbeat*
 

filebeat/docs/fields.asciidoc

@@ -89769,6 +89769,105 @@ kibana Module
 
 
 
+*`kibana.space_id`*::
++
+--
+The id of the space associated with this request.
+
+type: keyword
+
+example: default
+
+--
+
+*`kibana.saved_object.type`*::
++
+--
+The type of the saved object associated with this request.
+
+type: keyword
+
+example: dashboard
+
+--
+
+*`kibana.saved_object.id`*::
++
+--
+The id of the saved object associated with this request.
+
+type: keyword
+
+example: 6295bdd0-0a0e-11e7-825f-6748cda7d858
+
+--
+
+*`kibana.add_to_spaces`*::
++
+--
+The set of space ids that a saved object was shared to.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.delete_from_spaces`*::
++
+--
+The set of space ids that a saved object was removed from.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.authentication_provider`*::
++
+--
+The authentication provider associated with a login event.
+
+type: keyword
+
+example: basic1
+
+--
+
+*`kibana.authentication_type`*::
++
+--
+The authentication provider type associated with a login event.
+
+type: keyword
+
+example: basic
+
+--
+
+*`kibana.authentication_realm`*::
++
+--
+The Elasticsearch authentication realm name which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
+*`kibana.lookup_realm`*::
++
+--
+The Elasticsearch lookup realm which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
 [float]
 === log
 

filebeat/docs/modules/kibana.asciidoc

@@ -38,6 +38,11 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
 
 
 [float]

filebeat/filebeat.reference.yml

@@ -203,14 +203,22 @@ filebeat.modules:
 
 #-------------------------------- Kibana Module --------------------------------
 - module: kibana
- # All logs
+ # Server logs
  log:
  enabled: true
 
  # Set custom paths for the log files. If left empty,
  # Filebeat will choose the paths depending on your OS.
  #var.paths:
 
+ # Audit logs
+ audit:
+ enabled: true
+
+ # Set custom paths for the log files. If left empty,
+ # Filebeat will choose the paths depending on your OS.
+ #var.paths:
+
 #------------------------------- Logstash Module -------------------------------
 #- module: logstash
  # logs

filebeat/module/kibana/_meta/config.yml

@@ -1,8 +1,16 @@
 - module: kibana
- # All logs
+ # Server logs
  log:
  enabled: true
 
  # Set custom paths for the log files. If left empty,
  # Filebeat will choose the paths depending on your OS.
  #var.paths:
+
+ # Audit logs
+ audit:
+ enabled: true
+
+ # Set custom paths for the log files. If left empty,
+ # Filebeat will choose the paths depending on your OS.
+ #var.paths:

filebeat/module/kibana/_meta/docs.asciidoc

@@ -33,3 +33,8 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+

filebeat/module/kibana/_meta/fields.yml

@@ -5,5 +5,41 @@
  fields:
  - name: kibana
  type: group
- description: >
+ description: ''
  fields:
+ - name: space_id
+ description: "The id of the space associated with this request."
+ example: "default"
+ type: keyword
+ - name: saved_object.type
+ description: "The type of the saved object associated with this request."
+ example: "dashboard"
+ type: keyword
+ - name: saved_object.id
+ description: "The id of the saved object associated with this request."
+ example: "6295bdd0-0a0e-11e7-825f-6748cda7d858"
+ type: keyword
+ - name: add_to_spaces
+ description: "The set of space ids that a saved object was shared to."
+ example: "['default', 'marketing']"
+ type: keyword
+ - name: delete_from_spaces
+ description: "The set of space ids that a saved object was removed from."
+ example: "['default', 'marketing']"
+ type: keyword
+ - name: authentication_provider
+ description: "The authentication provider associated with a login event."
+ example: "basic1"
+ type: keyword
+ - name: authentication_type
+ description: "The authentication provider type associated with a login event."
+ example: "basic"
+ type: keyword
+ - name: authentication_realm
+ description: "The Elasticsearch authentication realm name which fulfilled a login event."
+ example: "native"
+ type: keyword
+ - name: lookup_realm
+ description: "The Elasticsearch lookup realm which fulfilled a login event."
+ example: "native"
+ type: keyword

filebeat/module/kibana/audit/config/audit.yml

@@ -0,0 +1,13 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+processors:
+ - add_locale: ~
+ - add_fields:
+ target: ''
+ fields:
+ ecs.version: 1.6.0

filebeat/module/kibana/audit/ingest/pipeline-json.yml

@@ -0,0 +1,88 @@
+description: Pipeline for parsing Kibana audit logs in JSON format
+processors:
+- json:
+ field: message
+ target_field: kibana._audit_temp
+
+- set:
+ field: "@timestamp"
+ value: "{{kibana._audit_temp.@timestamp}}"
+
+- set:
+ field: message
+ value: "{{kibana._audit_temp.message}}"
+
+- set:
+ if: ctx.kibana._audit_temp.event.action != null
+ field: event.action
+ value: "{{kibana._audit_temp.event.action}}"
+- set:
+ if: ctx.kibana._audit_temp.event.category != null
+ field: event.category
+ value: "{{kibana._audit_temp.event.category}}"
+- set:
+ if: ctx.kibana._audit_temp.event.outcome != null
+ field: event.outcome
+ value: "{{kibana._audit_temp.event.outcome}}"
+
+- rename:
+ if: ctx.kibana._audit_temp.url != null
+ field: kibana._audit_temp.url
+ target_field: "url"
+
+- set:
+ if: ctx.url?.query == null
+ field: url.original
+ value: '{{url.path}}'
+ ignore_empty_value: true
+- set:
+ if: ctx.url?.path != null && ctx.url?.query != null
+ field: url.original
+ value: '{{url.path}}?{{url.query}}'
+
+- rename:
+ if: ctx.kibana._audit_temp.http != null
+ field: kibana._audit_temp.http
+ target_field: http
+
+- rename:
+ if: ctx.kibana._audit_temp.user != null
+ field: kibana._audit_temp.user
+ target_field: user
+
+- rename:
+ if: ctx.kibana._audit_temp.trace != null
+ field: kibana._audit_temp.trace
+ target_field: trace
+
+- rename:
+ if: ctx.kibana._audit_temp.process?.pid != null
+ target_field: process
+ field: kibana._audit_temp.process
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.space_id != null
+ target_field: kibana.space_id
+ field: kibana._audit_temp.kibana.space_id
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.saved_object != null
+ target_field: kibana.saved_object
+ field: kibana._audit_temp.kibana.saved_object
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.add_to_spaces != null
+ target_field: kibana.add_to_spaces
+ field: kibana._audit_temp.kibana.add_to_spaces
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.delete_from_spaces != null
+ target_field: kibana.delete_from_spaces
+ field: kibana._audit_temp.kibana.delete_from_spaces
+
+- remove:
+ field: 'kibana._audit_temp'
+on_failure:
+- set:
+ field: error.message
+ value: '{{ _ingest.on_failure_message }}'

filebeat/module/kibana/audit/ingest/pipeline.yml

@@ -0,0 +1,21 @@
+description: Pipeline for parsing Kibana audit logs
+processors:
+- set:
+ field: event.ingested
+ value: '{{_ingest.timestamp}}'
+- rename:
+ field: '@timestamp'
+ target_field: event.created
+- pipeline:
+ name: '{< IngestPipeline "pipeline-json" >}'
+- set:
+ field: event.kind
+ value: event
+- append:
+ field: related.user
+ value: "{{user.name}}"
+ if: "ctx?.user?.name != null"
+on_failure:
+- set:
+ field: error.message
+ value: '{{ _ingest.on_failure_message }}'

filebeat/module/kibana/audit/manifest.yml

@@ -0,0 +1,12 @@
+module_version: 1.0
+
+var:
+ - name: paths
+ default:
+ - /var/log/kibana/*_audit.json
+
+ingest_pipeline:
+ - ingest/pipeline.yml
+ - ingest/pipeline-json.yml
+
+input: config/audit.yml

filebeat/module/kibana/audit/test/test-audit-711.log

@@ -0,0 +1,7 @@
+{"@timestamp":"2020-11-20T12:05:14.528-05:00","message":"User is updating config [id=8.0.0]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"saved_object_update","category":"database","type":"change","outcome":"unknown"},"kibana":{"space_id":"marketing","saved_object":{"type":"config","id":"8.0.0"}},"user":{"name":"elastic","roles":["superuser"]},"trace":{"id":"e0bd67a1-a1b0-424d-9652-a350f88188eb"}}
+{"@timestamp":"2020-11-20T12:05:14.849-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/_find] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/_find","port":5603,"query":"default_search_operator=AND&has_reference=%5B%5D&page=1&per_page=1000&search_fields=title%5E3&search_fields=description&type=dashboard","scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"ae67a156-3847-4d89-9c97-86769df5bc2e"}}
+{"@timestamp":"2020-11-20T12:05:15.841-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/_bulk_get] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/_bulk_get","port":5603,"scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"cef382d1-7442-4f9a-8bee-0512c2b1da5a"}}
+{"@timestamp":"2020-11-20T12:05:15.853-05:00","message":"User has accessed index-pattern [id=metrics-*]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"saved_object_get","category":"database","type":"access","outcome":"success"},"kibana":{"space_id":"marketing","saved_object":{"type":"index-pattern","id":"metrics-*"}},"user":{"name":"elastic","roles":["superuser"]},"trace":{"id":"cef382d1-7442-4f9a-8bee-0512c2b1da5a"}}
+{"@timestamp":"2020-11-20T12:05:24.103-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/_find] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/_find","port":5603,"query":"fields=title&per_page=10&search=%22My%20Dashboard%22&search_fields=title&type=dashboard","scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"b10ee3ab-8102-4122-b4b5-5727e9b3d6a3"}}
+{"@timestamp":"2020-11-20T12:05:24.143-05:00","message":"User is requesting [/foo/s/marketing/api/saved_objects/dashboard] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"http_request","category":"web","outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"0.0.0.0","path":"/foo/s/marketing/api/saved_objects/dashboard","port":5603,"query":"overwrite=true","scheme":"https:"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"marketing"},"trace":{"id":"4995c6bd-903c-42c2-af28-5cf17cc1cb6b"}}
+{"@timestamp":"2020-11-20T12:05:24.150-05:00","message":"User is creating dashboard [id=undefined]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":86516},"event":{"action":"saved_object_create","category":"database","type":"creation","outcome":"unknown"},"kibana":{"space_id":"marketing","saved_object":{"type":"dashboard"}},"user":{"name":"elastic","roles":["superuser"]},"trace":{"id":"4995c6bd-903c-42c2-af28-5cf17cc1cb6b"}}