#25480: Additional parsing for haproxy.http.request.raw_request_line

elastic · May 1, 2021 · c59eabb · c59eabb
1 parent 23c546e
commit c59eabb
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -833,6 +833,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803]
 - Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
 - New module `zookeeper` for Zookeeper service and audit logs {issue}25061[25061] {pull}25128[25128]
+- Add parsing for `haproxy.http.request.raw_request_line` field  {issue}25480[25480] {pull}25482[25482]
 
 *Heartbeat*
 

diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml
@@ -57,6 +57,15 @@ processors:
     field: haproxy.request_date
 - remove:
     field: message
+- grok:
+    field: haproxy.http.request.raw_request_line
+    patterns:
+      - '%{WORD:http.request.method}%{SPACE}%{URIPATHPARAM:url.original}%{SPACE}HTTP/%{NUMBER:http.version}'
+    ignore_missing: true
+- uri_parts:
+    field: url.original
+    ignore_failure: true
+    if: ctx?.url?.original != null
 - grok:
     field: source.address
     ignore_failure: true
@@ -135,6 +144,30 @@ processors:
     field: event.outcome
     value: failure
     if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
+- script:
+    lang: painless
+    description: This script processor iterates over the whole document to remove fields with null values.
+    source: |
+      void handleMap(Map map) {
+        for (def x : map.values()) {
+          if (x instanceof Map) {
+              handleMap(x);
+          } else if (x instanceof List) {
+              handleList(x);
+          }
+        }
+        map.values().removeIf(v -> v == null);
+      }
+      void handleList(List list) {
+        for (def x : list) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+        }
+      }
+      handleMap(ctx);
 on_failure:
 - set:
     field: error.message

diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json
@@ -33,8 +33,10 @@
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "----",
         "haproxy.total_waiting_time_ms": 0,
+        "http.request.method": "GET",
         "http.response.bytes": 168,
         "http.response.status_code": 304,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
         "process.name": "haproxy",
@@ -53,6 +55,9 @@
         "source.geo.region_iso_code": "RU-MOW",
         "source.geo.region_name": "Moscow",
         "source.ip": "1.2.3.4",
-        "source.port": 38862
+        "source.port": 38862,
+        "url.extension": "js",
+        "url.original": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js",
+        "url.path": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js"
     }
 ]
diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json
@@ -29,8 +29,10 @@
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "SC--",
         "haproxy.total_waiting_time_ms": -1,
+        "http.request.method": "GET",
         "http.response.bytes": 213,
         "http.response.status_code": 503,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
         "process.name": "haproxy",
@@ -41,7 +43,9 @@
         "service.type": "haproxy",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
-        "source.port": 35982
+        "source.port": 35982,
+        "url.original": "/",
+        "url.path": "/"
     },
     {
         "event.category": [
@@ -73,8 +77,10 @@
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "SC--",
         "haproxy.total_waiting_time_ms": -1,
+        "http.request.method": "GET",
         "http.response.bytes": 213,
         "http.response.status_code": 503,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 186,
         "process.name": "haproxy",
@@ -85,7 +91,9 @@
         "service.type": "haproxy",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
-        "source.port": 43738
+        "source.port": 43738,
+        "url.original": "/foo",
+        "url.path": "/foo"
     },
     {
         "event.category": [
@@ -121,8 +129,10 @@
         "haproxy.server_queue": 0,
         "haproxy.termination_state": "SC--",
         "haproxy.total_waiting_time_ms": -1,
+        "http.request.method": "GET",
         "http.response.bytes": 213,
         "http.response.status_code": 503,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 394,
         "process.name": "haproxy",
@@ -133,6 +143,8 @@
         "service.type": "haproxy",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
-        "source.port": 44542
+        "source.port": 44542,
+        "url.original": "/foo",
+        "url.path": "/foo"
     }
 ]