[Filebeat] Add module for Kibana audit logs (#22696) (#23147)

Co-authored-by: Thom Heymann <[email protected]>

Co-authored-by: Thom Heymann <[email protected]>

CHANGELOG.next.asciidoc

@@ -36,6 +36,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - Improve ECS field mappings in panw module. event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
 - Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]

filebeat/docs/fields.asciidoc

@@ -91044,9 +91044,120 @@ kibana Module
 [float]
 === kibana
 
+Module for parsing Kibana logs.
 
 
 
+*`kibana.session_id`*::
++
+--
+The ID of the user session associated with this event. Each login attempt results in a unique session id.
+
+type: keyword
+
+example: 123e4567-e89b-12d3-a456-426614174000
+
+--
+
+*`kibana.space_id`*::
++
+--
+The id of the space associated with this event.
+
+type: keyword
+
+example: default
+
+--
+
+*`kibana.saved_object.type`*::
++
+--
+The type of the saved object associated with this event.
+
+type: keyword
+
+example: dashboard
+
+--
+
+*`kibana.saved_object.id`*::
++
+--
+The id of the saved object associated with this event.
+
+type: keyword
+
+example: 6295bdd0-0a0e-11e7-825f-6748cda7d858
+
+--
+
+*`kibana.add_to_spaces`*::
++
+--
+The set of space ids that a saved object was shared to.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.delete_from_spaces`*::
++
+--
+The set of space ids that a saved object was removed from.
+
+type: keyword
+
+example: ['default', 'marketing']
+
+--
+
+*`kibana.authentication_provider`*::
++
+--
+The authentication provider associated with a login event.
+
+type: keyword
+
+example: basic1
+
+--
+
+*`kibana.authentication_type`*::
++
+--
+The authentication provider type associated with a login event.
+
+type: keyword
+
+example: basic
+
+--
+
+*`kibana.authentication_realm`*::
++
+--
+The Elasticsearch authentication realm name which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
+*`kibana.lookup_realm`*::
++
+--
+The Elasticsearch lookup realm which fulfilled a login event.
+
+type: keyword
+
+example: native
+
+--
+
 [float]
 === log
 

filebeat/docs/modules/kibana.asciidoc

@@ -38,6 +38,11 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
 
 
 [float]

filebeat/filebeat.reference.yml

@@ -203,14 +203,22 @@ filebeat.modules:
 
 #-------------------------------- Kibana Module --------------------------------
 - module: kibana
- # All logs
+ # Server logs
  log:
  enabled: true
 
  # Set custom paths for the log files. If left empty,
  # Filebeat will choose the paths depending on your OS.
  #var.paths:
 
+ # Audit logs
+ audit:
+ enabled: true
+
+ # Set custom paths for the log files. If left empty,
+ # Filebeat will choose the paths depending on your OS.
+ #var.paths:
+
 #------------------------------- Logstash Module -------------------------------
 #- module: logstash
  # logs

filebeat/module/kibana/_meta/config.yml

@@ -1,8 +1,16 @@
 - module: kibana
- # All logs
+ # Server logs
  log:
  enabled: true
 
  # Set custom paths for the log files. If left empty,
  # Filebeat will choose the paths depending on your OS.
  #var.paths:
+
+ # Audit logs
+ audit:
+ enabled: true
+
+ # Set custom paths for the log files. If left empty,
+ # Filebeat will choose the paths depending on your OS.
+ #var.paths:

filebeat/module/kibana/_meta/docs.asciidoc

@@ -33,3 +33,8 @@ include::../include/var-paths.asciidoc[]
 :fileset_ex!:
 
 :modulename!:
+[float]
+==== `audit` fileset settings
+
+include::../include/var-paths.asciidoc[]
+

filebeat/module/kibana/_meta/fields.yml

@@ -6,4 +6,45 @@
  - name: kibana
  type: group
  description: >
+ Module for parsing Kibana logs.
  fields:
+ - name: session_id
+ description: The ID of the user session associated with this event. Each login attempt results in a unique session id.
+ example: "123e4567-e89b-12d3-a456-426614174000"
+ type: keyword
+ - name: space_id
+ description: "The id of the space associated with this event."
+ example: "default"
+ type: keyword
+ - name: saved_object.type
+ description: "The type of the saved object associated with this event."
+ example: "dashboard"
+ type: keyword
+ - name: saved_object.id
+ description: "The id of the saved object associated with this event."
+ example: "6295bdd0-0a0e-11e7-825f-6748cda7d858"
+ type: keyword
+ - name: add_to_spaces
+ description: "The set of space ids that a saved object was shared to."
+ example: "['default', 'marketing']"
+ type: keyword
+ - name: delete_from_spaces
+ description: "The set of space ids that a saved object was removed from."
+ example: "['default', 'marketing']"
+ type: keyword
+ - name: authentication_provider
+ description: "The authentication provider associated with a login event."
+ example: "basic1"
+ type: keyword
+ - name: authentication_type
+ description: "The authentication provider type associated with a login event."
+ example: "basic"
+ type: keyword
+ - name: authentication_realm
+ description: "The Elasticsearch authentication realm name which fulfilled a login event."
+ example: "native"
+ type: keyword
+ - name: lookup_realm
+ description: "The Elasticsearch lookup realm which fulfilled a login event."
+ example: "native"
+ type: keyword

filebeat/module/kibana/audit/config/audit.yml

@@ -0,0 +1,16 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+processors:
+ - add_locale: ~
+ - add_fields:
+ target: ''
+ fields:
+ ecs.version: 0.0.0
+ - decode_json_fields:
+ fields: [message]
+ target: kibana._audit_temp

filebeat/module/kibana/audit/ingest/pipeline-json.yml

@@ -0,0 +1,106 @@
+description: Pipeline for parsing Kibana audit logs in JSON format
+processors:
+
+- set:
+ field: "@timestamp"
+ value: "{{kibana._audit_temp.@timestamp}}"
+
+- set:
+ field: message
+ value: "{{kibana._audit_temp.message}}"
+
+- set:
+ if: ctx.kibana._audit_temp.event.action != null
+ field: event.action
+ value: "{{kibana._audit_temp.event.action}}"
+- set:
+ if: ctx.kibana._audit_temp.event.category != null
+ field: event.category
+ value: "{{kibana._audit_temp.event.category}}"
+- set:
+ if: ctx.kibana._audit_temp.event.outcome != null
+ field: event.outcome
+ value: "{{kibana._audit_temp.event.outcome}}"
+- set:
+ if: ctx.kibana._audit_temp.event.type != null
+ field: event.type
+ value: "{{kibana._audit_temp.event.type}}"
+
+- remove:
+ field: 'ecs'
+- rename:
+ if: ctx.kibana._audit_temp.ecs != null
+ field: kibana._audit_temp.ecs
+ target_field: "ecs"
+
+- rename:
+ if: ctx.kibana._audit_temp.url != null
+ field: kibana._audit_temp.url
+ target_field: "url"
+
+- set:
+ if: ctx.url?.query == null
+ field: url.original
+ value: '{{url.path}}'
+ ignore_empty_value: true
+- set:
+ if: ctx.url?.path != null && ctx.url?.query != null
+ field: url.original
+ value: '{{url.path}}?{{url.query}}'
+
+- rename:
+ if: ctx.kibana._audit_temp.http != null
+ field: kibana._audit_temp.http
+ target_field: http
+
+- rename:
+ if: ctx.kibana._audit_temp.user != null
+ field: kibana._audit_temp.user
+ target_field: user
+
+- rename:
+ if: ctx.kibana._audit_temp.trace != null
+ field: kibana._audit_temp.trace
+ target_field: trace
+
+- rename:
+ if: ctx.kibana._audit_temp.process?.pid != null
+ target_field: process
+ field: kibana._audit_temp.process
+
+- rename:
+ if: ctx.kibana._audit_temp.error != null
+ target_field: error
+ field: kibana._audit_temp.error
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.session_id != null
+ target_field: kibana.session_id
+ field: kibana._audit_temp.kibana.session_id
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.space_id != null
+ target_field: kibana.space_id
+ field: kibana._audit_temp.kibana.space_id
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.saved_object != null
+ target_field: kibana.saved_object
+ field: kibana._audit_temp.kibana.saved_object
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.add_to_spaces != null
+ target_field: kibana.add_to_spaces
+ field: kibana._audit_temp.kibana.add_to_spaces
+
+- rename:
+ if: ctx.kibana._audit_temp.kibana.delete_from_spaces != null
+ target_field: kibana.delete_from_spaces
+ field: kibana._audit_temp.kibana.delete_from_spaces
+
+- remove:
+ field: 'kibana._audit_temp'
+on_failure:
+- set:
+ field: error.message
+ value: '{{ _ingest.on_failure_message }}'

filebeat/module/kibana/audit/ingest/pipeline.yml

@@ -0,0 +1,21 @@
+description: Pipeline for parsing Kibana audit logs
+processors:
+- set:
+ field: event.ingested
+ value: '{{_ingest.timestamp}}'
+- rename:
+ field: '@timestamp'
+ target_field: event.created
+- pipeline:
+ name: '{< IngestPipeline "pipeline-json" >}'
+- set:
+ field: event.kind
+ value: event
+- append:
+ field: related.user
+ value: "{{user.name}}"
+ if: "ctx?.user?.name != null"
+on_failure:
+- set:
+ field: error.message
+ value: '{{ _ingest.on_failure_message }}'

filebeat/module/kibana/audit/manifest.yml

@@ -0,0 +1,12 @@
+module_version: 1.0
+
+var:
+ - name: paths
+ default:
+ - /var/log/kibana/*_audit.json
+
+ingest_pipeline:
+ - ingest/pipeline.yml
+ - ingest/pipeline-json.yml
+
+input: config/audit.yml