[8.14](backport #40446) [syslog] Fix handling of escaped characters i…

…n structured data (#40463)

* [syslog] Fix handling of escaped characters in structured data (#40446)

- Improved parser to handle escaped closing square brackets in structured data, along
with square brackets in the normal, non-structured data portion of the message.
- Fix incorrect offset being passed to removeBytes function, which would not remove
escaped characters from structured data values.
- The non-compliant-sd unit test cases now include escapes on the closing brackets
within the structured data, something that should have always been there.
- Add tests

(cherry picked from commit 1c01d0e)

* fix up changelog

---------

Co-authored-by: Taylor Swanson <[email protected]>
Co-authored-by: Taylor Swanson <[email protected]>

CHANGELOG.next.asciidoc

@@ -62,6 +62,7 @@ https:/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
+- Fix handling of escaped brackets in syslog structured data. {issue}40445[40445] {pull}40446[40446]
 
 *Auditbeat*
 

libbeat/reader/syslog/message_test.go

@@ -522,6 +522,20 @@ func TestParseStructuredData(t *testing.T) {
  },
  },
  },
+ "multi-key-with-escape": {
+ in: `[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011" somekey="[value\] more data"][examplePriority@32473 class="high"]`,
+ want: map[string]interface{}{
+ "exampleSDID@32473": map[string]interface{}{
+ "iut": "3",
+ "eventSource": "Application",
+ "eventID": "1011",
+ "somekey": "[value] more data",
+ },
+ "examplePriority@32473": map[string]interface{}{
+ "class": "high",
+ },
+ },
+ },
  "repeated-id": {
  in: `[exampleSDID@32473 iut="3"][exampleSDID@32473 class="high"]`,
  want: map[string]interface{}{

libbeat/reader/syslog/parser/rfc5424.rl

@@ -36,7 +36,7 @@
 
  action set_param_value {
  if subMap, ok := structuredData[s.sdID].(map[string]interface{}); ok {
- subMap[s.sdParamName] = removeBytes(data[tok:p], s.sdValueEscapes, p)
+ subMap[s.sdParamName] = removeBytes(data[tok:p], s.sdValueEscapes, tok)
  }
  }
 
@@ -73,7 +73,9 @@
 
  header = priority version sp timestamp sp hostname sp app_name sp proc_id sp msg_id;
 
- sd_raw = nil_value | ('[' any+ ']') >tok %set_sd_raw;
+ sd_raw_escape = (bs | ']');
+ sd_raw_values = ((bs ']') | (any - sd_raw_escape));
+ sd_raw = nil_value | ('[' sd_raw_values+ ']')+ >tok %set_sd_raw;
 
  msg = any* >tok %set_msg;
 }%%