[syslog] Fix handling of escaped characters in structured data (#40446)

- Improved parser to handle escaped closing square brackets in structured data, along
with square brackets in the normal, non-structured data portion of the message.
- Fix incorrect offset being passed to removeBytes function, which would not remove
escaped characters from structured data values.
- The non-compliant-sd unit test cases now include escapes on the closing brackets
within the structured data, something that should have always been there.
- Add tests

(cherry picked from commit 1c01d0e)

CHANGELOG.next.asciidoc

@@ -68,6 +68,14 @@ https:/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
+- Fix the paths in the .cmd script added to the path by the Windows MSI to point to the new C:\Program Files installation location. https:/elastic/elastic-stack-installers/pull/238
+- Change cache processor documentation from `write_period` to `write_interval`. {pull}38561[38561]
+- Fix cache processor expiries heap cleanup on partial file writes. {pull}38561[38561]
+- Fix cache processor expiries infinite growth when large a large TTL is used and recurring keys are cached. {pull}38561[38561]
+- Fix parsing of RFC 3164 process IDs in syslog processor. {issue}38947[38947] {pull}38982[38982]
+- Rename the field "apache2.module.error" to "apache.module.error" in Apache error visualization. {issue}39480[39480] {pull}39481[39481]
+- Validate config of the `replace` processor {pull}40047[40047]
+- Fix handling of escaped brackets in syslog structured data. {issue}40445[40445] {pull}40446[40446]
 
 *Auditbeat*
 

libbeat/reader/syslog/message_test.go

@@ -522,6 +522,20 @@ func TestParseStructuredData(t *testing.T) {
  },
  },
  },
+ "multi-key-with-escape": {
+ in: `[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011" somekey="[value\] more data"][examplePriority@32473 class="high"]`,
+ want: map[string]interface{}{
+ "exampleSDID@32473": map[string]interface{}{
+ "iut": "3",
+ "eventSource": "Application",
+ "eventID": "1011",
+ "somekey": "[value] more data",
+ },
+ "examplePriority@32473": map[string]interface{}{
+ "class": "high",
+ },
+ },
+ },
  "repeated-id": {
  in: `[exampleSDID@32473 iut="3"][exampleSDID@32473 class="high"]`,
  want: map[string]interface{}{

libbeat/reader/syslog/parser/rfc5424.rl

@@ -36,7 +36,7 @@
 
  action set_param_value {
  if subMap, ok := structuredData[s.sdID].(map[string]interface{}); ok {
- subMap[s.sdParamName] = removeBytes(data[tok:p], s.sdValueEscapes, p)
+ subMap[s.sdParamName] = removeBytes(data[tok:p], s.sdValueEscapes, tok)
  }
  }
 
@@ -73,7 +73,9 @@
 
  header = priority version sp timestamp sp hostname sp app_name sp proc_id sp msg_id;
 
- sd_raw = nil_value | ('[' any+ ']') >tok %set_sd_raw;
+ sd_raw_escape = (bs | ']');
+ sd_raw_values = ((bs ']') | (any - sd_raw_escape));
+ sd_raw = nil_value | ('[' sd_raw_values+ ']')+ >tok %set_sd_raw;
 
  msg = any* >tok %set_msg;
 }%%