[syslog] Fix handling of escaped characters in structured data (#40446)

- Improved parser to handle escaped closing square brackets in structured data, along
with square brackets in the normal, non-structured data portion of the message.
- Fix incorrect offset being passed to removeBytes function, which would not remove
escaped characters from structured data values.
- The non-compliant-sd unit test cases now include escapes on the closing brackets
within the structured data, something that should have always been there.
- Add tests

(cherry picked from commit 1c01d0e)

CHANGELOG.next.asciidoc

@@ -68,6 +68,14 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
+- Fix the paths in the .cmd script added to the path by the Windows MSI to point to the new C:\Program Files installation location. https://github.com/elastic/elastic-stack-installers/pull/238
+- Change cache processor documentation from `write_period` to `write_interval`. {pull}38561[38561]
+- Fix cache processor expiries heap cleanup on partial file writes. {pull}38561[38561]
+- Fix cache processor expiries infinite growth when large a large TTL is used and recurring keys are cached. {pull}38561[38561]
+- Fix parsing of RFC 3164 process IDs in syslog processor. {issue}38947[38947] {pull}38982[38982]
+- Rename the field "apache2.module.error" to "apache.module.error" in Apache error visualization. {issue}39480[39480] {pull}39481[39481]
+- Validate config of the `replace` processor {pull}40047[40047]
+- Fix handling of escaped brackets in syslog structured data. {issue}40445[40445] {pull}40446[40446]
 
 *Auditbeat*
 

libbeat/reader/syslog/message_test.go

@@ -522,6 +522,20 @@ func TestParseStructuredData(t *testing.T) {
 				},
 			},
 		},
+		"multi-key-with-escape": {
+			in: `[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011" somekey="[value\] more data"][examplePriority@32473 class="high"]`,
+			want: map[string]interface{}{
+				"exampleSDID@32473": map[string]interface{}{
+					"iut":         "3",
+					"eventSource": "Application",
+					"eventID":     "1011",
+					"somekey":     "[value] more data",
+				},
+				"examplePriority@32473": map[string]interface{}{
+					"class": "high",
+				},
+			},
+		},
 		"repeated-id": {
 			in: `[exampleSDID@32473 iut="3"][exampleSDID@32473 class="high"]`,
 			want: map[string]interface{}{

libbeat/reader/syslog/parser/rfc5424.rl

@@ -36,7 +36,7 @@
 
     action set_param_value {
         if subMap, ok := structuredData[s.sdID].(map[string]interface{}); ok {
-            subMap[s.sdParamName] = removeBytes(data[tok:p], s.sdValueEscapes, p)
+            subMap[s.sdParamName] = removeBytes(data[tok:p], s.sdValueEscapes, tok)
         }
     }
 
@@ -73,7 +73,9 @@
 
     header = priority version sp timestamp sp hostname sp app_name sp proc_id sp msg_id;
 
-    sd_raw = nil_value | ('[' any+ ']') >tok %set_sd_raw;
+    sd_raw_escape = (bs | ']');
+    sd_raw_values = ((bs ']') | (any - sd_raw_escape));
+    sd_raw        = nil_value | ('[' sd_raw_values+ ']')+ >tok %set_sd_raw;
 
     msg = any* >tok %set_msg;
 }%%