Log raw events to a separate log file

NOTICE.txt

@@ -2643,6 +2643,29 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/
  limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/elastic/mock-es
+Version: v0.0.0-20240605193845-b5546a703d6f
+Licence type (autodetected): Apache-2.0
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
+
+Copyright 2024 Elasticsearch B.V.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/fatih/color
 Version: v1.15.0
@@ -6214,6 +6237,45 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/rcrowley/go-metrics
+Version: v0.0.0-20201227073835-cf1acfcdf475
+Licence type (autodetected): BSD-2-Clause-FreeBSD
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/rcrowley/[email protected]/LICENSE:
+
+Copyright 2012 Richard Crowley. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation
+are those of the authors and should not be interpreted as representing
+official policies, either expressed or implied, of Richard Crowley.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/rs/zerolog
 Version: v1.27.0
@@ -16542,11 +16604,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.14.3
+Version: v0.14.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE:
 
  Apache License
  Version 2.0, January 2004
@@ -22301,6 +22363,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/mileusna/useragent
+Version: v1.3.4
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/mileusna/[email protected]/LICENSE.md:
+
+MIT License
+
+Copyright (c) 2017 Miloš Mileusnić
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 --------------------------------------------------------------------------------
 Dependency : github.com/mitchellh/colorstring
 Version: v0.0.0-20190213212951-d06e56a500db

_meta/config/common.reference.p2.yml.tmpl

@@ -272,4 +272,52 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
+#=============================== Events Logging ===============================
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events (that may contain
+# sensitive information) together with other log messages, a different
+# log file, only for log entries containing raw events, is used. It will
+# use the same level, selectors and all other configurations from the
+# default logger, but it will have it's own file configuration.
+#
+# Having a different log file for raw events also prevents event data
+# from drowning out the regular log files.
+#
+# IMPORTANT: No matter the default logger output configuration, raw events
+# will **always** be logged to a file configured by `agent.logging.event_data.files`.
+
+# agent.logging.event_data:
+# Logging to rotating files. Set agent.logging.to_files to false to disable logging to
+# files.
+#agent.logging.event_data.to_files: true
+#agent.logging.event_data:
+ # Configure the path where the logs are written. The default is the logs directory
+ # under the home path (the binary location).
+ #path: /var/log/filebeat
+
+ # The name of the files where the logs are written to.
+ #name: filebeat-event-data
+
+ # Configure log file size limit. If the limit is reached, log file will be
+ # automatically rotated.
+ #rotateeverybytes: 5242880 # = 5MB
+
+ # Number of rotated log files to keep. The oldest files will be deleted first.
+ #keepfiles: 2
+
+ # The permissions mask to apply when rotating log files. The default value is 0600.
+ # Must be a valid Unix-style file permissions mask expressed in octal notation.
+ #permissions: 0600
+
+ # Enable log file rotation on time intervals in addition to the size-based rotation.
+ # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+ # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+ # reported by the local system clock. All other intervals are calculated from the
+ # Unix epoch. Defaults to disabled.
+ #interval: 0
+
+ # Rotate existing logs on startup rather than appending them to the existing
+ # file. Defaults to false.
+ # rotateonstartup: false
+
 {{template "providers.yml.tmpl" .}}

changelog/fragments/1712935205-Log-raw-events-to-a-separate-log-file.yaml

@@ -0,0 +1,36 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Log raw events to a separate log file
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: |
+ Log entries from Elastic-Agent inputs containing event data are now
+ directed to a different file under the 'events' folder in the logs
+ directory. This file is not sent to the monitoring output, however
+ it can be retrieved by collecting a diagnostics.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https:/elastic/elastic-agent/pull/4549
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https:/owner/repo/1234

docs/elastic-agent-logging.md

@@ -44,6 +44,16 @@ configuration is:
  - ECS/JSON encoded
  - UTC timestamps
 
+There is also a second file output for events that is configured via
+`agent.logging.event_data`. It shares the same log level as the main
+logger and can only be configured for standalone agents. For
+Fleet-Managed agents it will always use the default values:
+ - 5Mb per log file
+ - Maximum of 2 log files
+ - Do not rotate on startup
+ - ECS/JSON encoded
+ - UTC timestamps
+
 ## Default logging
 The default logger is the easiest to discover because it's user
 configurable, logs to the Agent's root directory and can output to

elastic-agent.reference.yml

@@ -278,6 +278,54 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
+#=============================== Events Logging ===============================
+# Some outputs will log raw events on errors like indexing errors in the
+# Elasticsearch output, to prevent logging raw events (that may contain
+# sensitive information) together with other log messages, a different
+# log file, only for log entries containing raw events, is used. It will
+# use the same level, selectors and all other configurations from the
+# default logger, but it will have it's own file configuration.
+#
+# Having a different log file for raw events also prevents event data
+# from drowning out the regular log files.
+#
+# IMPORTANT: No matter the default logger output configuration, raw events
+# will **always** be logged to a file configured by `agent.logging.event_data.files`.
+
+# agent.logging.event_data:
+# Logging to rotating files. Set agent.logging.to_files to false to disable logging to
+# files.
+#agent.logging.event_data.to_files: true
+#agent.logging.event_data:
+ # Configure the path where the logs are written. The default is the logs directory
+ # under the home path (the binary location).
+ #path: /var/log/filebeat
+
+ # The name of the files where the logs are written to.
+ #name: filebeat-event-data
+
+ # Configure log file size limit. If the limit is reached, log file will be
+ # automatically rotated.
+ #rotateeverybytes: 5242880 # = 5MB
+
+ # Number of rotated log files to keep. The oldest files will be deleted first.
+ #keepfiles: 2
+
+ # The permissions mask to apply when rotating log files. The default value is 0600.
+ # Must be a valid Unix-style file permissions mask expressed in octal notation.
+ #permissions: 0600
+
+ # Enable log file rotation on time intervals in addition to the size-based rotation.
+ # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h
+ # are boundary-aligned with minutes, hours, days, weeks, months, and years as
+ # reported by the local system clock. All other intervals are calculated from the
+ # Unix epoch. Defaults to disabled.
+ #interval: 0
+
+ # Rotate existing logs on startup rather than appending them to the existing
+ # file. Defaults to false.
+ # rotateonstartup: false
+
 # Providers
 
 # Providers supply the key/values pairs that are used for variable substitution

go.mod

@@ -1,8 +1,6 @@
 module github.com/elastic/elastic-agent
 
-go 1.21.0
-
-toolchain go1.21.10
+go 1.21.10
 
 require (
  github.com/Microsoft/go-winio v0.6.1
@@ -23,6 +21,7 @@ require (
  github.com/elastic/go-licenser v0.4.1
  github.com/elastic/go-sysinfo v1.14.0
  github.com/elastic/go-ucfg v0.8.8
+ github.com/elastic/mock-es v0.0.0-20240605193845-b5546a703d6f
  github.com/fatih/color v1.15.0
  github.com/fsnotify/fsnotify v1.7.0
  github.com/gofrs/flock v0.8.1
@@ -46,6 +45,7 @@ require (
  github.com/otiai10/copy v1.14.0
  github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
  github.com/pkg/errors v0.9.1
+ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
  github.com/rs/zerolog v1.27.0
  github.com/sajari/regression v1.0.1
  github.com/schollz/progressbar/v3 v3.13.1
@@ -84,6 +84,8 @@ require (
 )
 
 require (
+ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect
+ github.com/distribution/reference v0.5.0 // indirect
  // open telemetry dependencies
  github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter v0.101.0
  github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.101.0
@@ -121,19 +123,17 @@ require (
  github.com/armon/go-radix v1.0.0 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
- github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect
  github.com/cespare/xxhash/v2 v2.3.0 // indirect
  github.com/cyphar/filepath-securejoin v0.2.4 // indirect
  github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
- github.com/distribution/reference v0.5.0 // indirect
  github.com/dnephin/pflag v1.0.7 // indirect
  github.com/docker/docker v25.0.5+incompatible // indirect
  github.com/docker/go-connections v0.5.0 // indirect
  github.com/elastic/go-docappender/v2 v2.1.2 // indirect
  github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
  github.com/elastic/go-structform v0.0.10 // indirect
  github.com/elastic/go-windows v1.0.1 // indirect
- github.com/elastic/gosigar v0.14.3 // indirect
+ github.com/elastic/gosigar v0.14.2 // indirect
  github.com/elastic/pkcs8 v1.0.0 // indirect
  github.com/emicklei/go-restful/v3 v3.10.1 // indirect
  github.com/evanphx/json-patch v5.6.0+incompatible // indirect
@@ -180,6 +180,7 @@ require (
  github.com/mattn/go-colorable v0.1.13 // indirect
  github.com/mattn/go-isatty v0.0.17 // indirect
  github.com/mattn/go-runewidth v0.0.15 // indirect
+ github.com/mileusna/useragent v1.3.4 // indirect
  github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
  github.com/mitchellh/copystructure v1.2.0 // indirect
  github.com/mitchellh/go-homedir v1.1.0 // indirect

go.sum

@@ -827,8 +827,10 @@ github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
 github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg=
-github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
-github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
+github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/mock-es v0.0.0-20240605193845-b5546a703d6f h1:qo0Nn9G+p7HbG+CmiWwSoiKJ/cyRq59TRn9Z2PRHTi8=
+github.com/elastic/mock-es v0.0.0-20240605193845-b5546a703d6f/go.mod h1:mVdKBYYwt30xRFjCegbcURHh+3LWOCkQM33fgWbUiRI=
 github.com/elastic/package-spec/v2 v2.6.0/go.mod h1:ks9/FaVOS+vCrGRQcDvXAd2FlmB84mrLikbRiO6ACuk=
 github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA=
 github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0=
@@ -1404,6 +1406,8 @@ github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKju
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mileusna/useragent v1.3.4 h1:MiuRRuvGjEie1+yZHO88UBYg8YBC/ddF6T7F56i3PCk=
+github.com/mileusna/useragent v1.3.4/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
@@ -1686,6 +1690,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua
 github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
 github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=