Cover empty request data, url and version in Apache2 Filebeat module (#…

…10730)
elastic · Feb 15, 2019 · 7c338ce · 7c338ce
1 parent 6c6df1b
commit 7c338ce
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 3 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -185,6 +185,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix bad bytes count in `docker` input when filtering by stream. {pull}10211[10211]
 - Fixed data types for roles and indices fields in `elasticsearch/audit` fileset {pull}10307[10307]
 - Ensure `source.address` is always populated by the nginx module (ECS). {pull}10418[10418]
+- Cover empty request data, url and version in Apache2 module{pull}10730[10730]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/access/ingest/default.json b/filebeat/module/apache/access/ingest/default.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?",
+        "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?",
         "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -",
         "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.body.bytes:long}"
         ],
@@ -31,11 +31,13 @@
     "date": {
       "field": "apache.access.time",
       "target_field": "@timestamp",
-      "formats": ["dd/MMM/yyyy:H:m:s Z"]
+      "formats": ["dd/MMM/yyyy:H:m:s Z"],
+        "ignore_failure": true
     }
   }, {
     "remove": {
-      "field": "apache.access.time"
+      "field": "apache.access.time",
+        "ignore_failure": true
     }
   }, {
     "user_agent": {

diff --git a/filebeat/module/apache/access/test/test.log b/filebeat/module/apache/access/test/test.log
@@ -3,3 +3,4 @@
 ::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 -
 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
 monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
+127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-"
diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json
@@ -104,5 +104,24 @@
         "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2",
         "user_agent.os.name": "Windows 7",
         "user_agent.version": "15.0.a2"
+    },
+    {
+        "@timestamp": "2019-02-02T04:38:45.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "apache.access",
+        "event.module": "apache",
+        "fileset.name": "access",
+        "http.request.referrer": "-",
+        "http.response.body.bytes": 152,
+        "http.response.status_code": 408,
+        "input.type": "log",
+        "log.offset": 603,
+        "service.type": "apache",
+        "source.address": "127.0.0.1",
+        "source.ip": "127.0.0.1",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Other",
+        "user_agent.original": "-"
     }
 ]