From fde0eb7f942202cb1f9efbd146f9abd60f6d3f97 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 9 May 2024 09:40:08 +0930 Subject: [PATCH 1/3] x-pack/filebeat/input/internal/httplog: improve req/resp logging (#39455) Attempt to log the request and response bodies and other details even when copying the body has been reported to have failed. (cherry picked from commit 3efb1e8612baae8b9b887afd0219c94caf36c7c1) --- CHANGELOG.next.asciidoc | 31 +++++++++++++++++++ .../input/internal/httplog/roundtripper.go | 26 +++++++--------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index caa91b893aa..55978417dd9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -129,6 +129,37 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999] - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128] - Made GCS input GA and updated docs accordingly. {pull}37127[37127] +- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160] +- Prevent CEL input from re-entering the eval loop when an evaluation failed. {pull}37161[37161] +- Update CEL extensions library to v1.7.0. {pull}37172[37172] +- Add support for complete URL replacement in HTTPJSON chain steps. {pull}37486[37486] +- Add support for user-defined query selection in EntraID entity analytics provider. {pull}37653[37653] +- Update CEL extensions library to v1.8.0 to provide runtime error location reporting. {issue}37304[37304] {pull}37718[37718] +- Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682] +- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714] +- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772] +- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] +- Added experimental version of the Websocket Input. {pull}37774[37774] +- Add support for PEM-based Okta auth in CEL. {pull}37813[37813] +- Add Salesforce input. {pull}37331[37331] +- Add ETW input. {pull}36915[36915] +- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] +- Add logging for cache processor file reads and writes. {pull}38052[38052] +- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] +- Support VPC endpoint for aws-s3 input SQS queue url. {pull}38189[38189] +- Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] +- Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] +- Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] +- Parse more fields from Elasticsearch slowlogs {pull}38295[38295] +- Update CEL mito extensions to v1.10.0 to add base64 decode functions. {pull}38504[38504] +- Add support for Active Directory an entity analytics provider. {pull}37919[37919] +- Add AWS AWSHealth metricset. {pull}38370[38370] +- Add debugging breadcrumb to logs when writing request trace log. {pull}38636[38636] +- added benchmark input {pull}37437[37437] +- added benchmark input and discard output {pull}37437[37437] +- Ensure all responses sent by HTTP Endpoint are HTML-escaped. {pull}39329[39329] +- Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460] +- Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455] *Auditbeat* diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index eac54d7378f..c313373fc30 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -109,14 +109,13 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err resp.Body, body, err = copyBody(resp.Body) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err)) - } else { - respParts = append(respParts, - zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]), - zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)), - zap.Int("http.response.body.bytes", len(body)), - zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), - ) } + respParts = append(respParts, + zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]), + zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)), + zap.Int("http.response.body.bytes", len(body)), + zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), + ) message, err := httputil.DumpResponse(resp, false) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump response: %s", err)) @@ -178,14 +177,13 @@ func logRequest(log *zap.Logger, req *http.Request, maxBodyLen int, extra ...zap req.Body, body, err = copyBody(req.Body) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err)) - } else { - reqParts = append(reqParts, - zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]), - zap.Bool("http.request.body.truncated", maxBodyLen < len(body)), - zap.Int("http.request.body.bytes", len(body)), - zap.String("http.request.mime_type", req.Header.Get("Content-Type")), - ) } + reqParts = append(reqParts, + zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]), + zap.Bool("http.request.body.truncated", maxBodyLen < len(body)), + zap.Int("http.request.body.bytes", len(body)), + zap.String("http.request.mime_type", req.Header.Get("Content-Type")), + ) message, err := httputil.DumpRequestOut(req, false) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump request: %s", err)) From 8dca55ece2e0cb0adbe6d430ae6a4119956c18f8 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 9 May 2024 09:51:46 +0930 Subject: [PATCH 2/3] remove irrelevant changelog entries --- CHANGELOG.next.asciidoc | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 55978417dd9..d3f2aff582b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -129,36 +129,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999] - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128] - Made GCS input GA and updated docs accordingly. {pull}37127[37127] -- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160] -- Prevent CEL input from re-entering the eval loop when an evaluation failed. {pull}37161[37161] -- Update CEL extensions library to v1.7.0. {pull}37172[37172] -- Add support for complete URL replacement in HTTPJSON chain steps. {pull}37486[37486] -- Add support for user-defined query selection in EntraID entity analytics provider. {pull}37653[37653] -- Update CEL extensions library to v1.8.0 to provide runtime error location reporting. {issue}37304[37304] {pull}37718[37718] -- Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682] -- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714] -- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772] -- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] -- Added experimental version of the Websocket Input. {pull}37774[37774] -- Add support for PEM-based Okta auth in CEL. {pull}37813[37813] -- Add Salesforce input. {pull}37331[37331] -- Add ETW input. {pull}36915[36915] -- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] -- Add logging for cache processor file reads and writes. {pull}38052[38052] -- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] -- Support VPC endpoint for aws-s3 input SQS queue url. {pull}38189[38189] -- Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] -- Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] -- Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] -- Parse more fields from Elasticsearch slowlogs {pull}38295[38295] -- Update CEL mito extensions to v1.10.0 to add base64 decode functions. {pull}38504[38504] -- Add support for Active Directory an entity analytics provider. {pull}37919[37919] -- Add AWS AWSHealth metricset. {pull}38370[38370] -- Add debugging breadcrumb to logs when writing request trace log. {pull}38636[38636] -- added benchmark input {pull}37437[37437] -- added benchmark input and discard output {pull}37437[37437] -- Ensure all responses sent by HTTP Endpoint are HTML-escaped. {pull}39329[39329] -- Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460] - Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455] *Auditbeat* From 84eba7347e4fe8872afa792d54547a1c66e407ab Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 9 May 2024 09:54:19 +0930 Subject: [PATCH 3/3] add pre go1.21 min helper --- x-pack/filebeat/input/internal/httplog/roundtripper.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index c313373fc30..97991928c68 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -202,6 +202,13 @@ func logRequest(log *zap.Logger, req *http.Request, maxBodyLen int, extra ...zap return req, reqParts[:0], errorsMessages } +func min(a, b int) int { + if a < b { + return a + } + return b +} + // TxID returns the current transaction.id value. If rt is nil, the empty string is returned. func (rt *LoggingRoundTripper) TxID() string { if rt == nil {