From 8927920e6273f4da5f5edec445fb760e7563c814 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 16:01:51 +0100 Subject: [PATCH] handling IPv6 within square brackets (backport #32989) (#33008) * handling IPv6 within square brackets (#32989) --- CHANGELOG.next.asciidoc | 1 + .../ingress_controller/ingest/pipeline.yml | 43 +++- .../nginx/ingress_controller/test/test.log | 3 + .../test/test.log-expected.json | 215 ++++++++++++++++++ 4 files changed, 253 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0f1169cef89..f3c0a4a31d5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Filebeat* - Fix a hang on `apt-get update` stage in packaging. {pull}32580[32580] +- Fix how to handle IPv6 addresses in the fileset `nginx/ingress_controller` for Filebeat. {pull}32989[32989] *Heartbeat* diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index eaed9342b61..4e682c0261e 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -27,6 +27,7 @@ processors: UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* + IP: (?:\[?%{IPV6}\]?|%{IPV4}) ignore_missing: true - grok: field: nginx.ingress_controller.info @@ -139,18 +140,42 @@ processors: for (def item : ctx.nginx.ingress_controller.upstream_address_list) { last_upstream = item; } - StringTokenizer tok = new StringTokenizer(last_upstream, ":"); - if (tok.countTokens()>1) { - ctx.nginx.ingress_controller.upstream.ip = tok.nextToken(); - ctx.nginx.ingress_controller.upstream.port = Integer.parseInt(tok.nextToken()); - } else { - ctx.nginx.ingress_controller.upstream.ip = last_upstream; - } + + ctx.nginx.ingress_controller.upstream.address = last_upstream; } catch (Exception e) { - ctx.nginx.ingress_controller.upstream.ip = null; - ctx.nginx.ingress_controller.upstream.port = null; + ctx.nginx.ingress_controller.upstream.address = null; } + - grok: + field: nginx.ingress_controller.upstream.address + patterns: + - "^%{IPV4:nginx.ingress_controller.upstream.ip}:%{NUMBER:nginx.ingress_controller.upstream.port}$" + - "^\\[%{IPV6:nginx.ingress_controller.upstream.ip}\\]:%{NUMBER:nginx.ingress_controller.upstream.port}$" + - "^%{IPV6NOCOMPRESS:nginx.ingress_controller.upstream.ip}:%{NUMBER:nginx.ingress_controller.upstream.port}$" + - "^%{IPV6:nginx.ingress_controller.upstream.ip}%{IPV6PORTSEP}%{NUMBER:nginx.ingress_controller.upstream.port}$" + - "^%{IPV6:nginx.ingress_controller.upstream.ip}%{IPV6PORTSEP}%{POSINT:nginx.ingress_controller.upstream.port}$" + pattern_definitions: + IPV6NOCOMPRESS: '([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' + IPV6PORTSEP: '(?: port |[p#.])' + ignore_missing: true + ignore_failure: true + - convert: + field: nginx.ingress_controller.upstream.ip + type: ip + ignore_missing: true + on_failure: + - remove: + field: nginx.ingress_controller.upstream.ip + - convert: + field: nginx.ingress_controller.upstream.port + type: long + ignore_missing: true + on_failure: + - remove: + field: nginx.ingress_controller.upstream.port + - remove: + field: nginx.ingress_controller.upstream.address + ignore_failure: true - script: if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 lang: painless diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index ef502ebe86b..1c80c68213b 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -22,3 +22,6 @@ 192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 +2a02:cf40:: - remote_monitoring_user [24/Aug/2022:21:04:17 +0000] "POST /_bulk HTTP/1.1" 200 470 "-" "Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)" 2057 0.033 [esmon-esmon-es-http-9200] [] [2a02:cf40::]:9200 470 0.036 200 3db73c6c673c4256ade033a6ce08c2ab +2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::4e36]:5000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab +2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::::::::4e36]:500000000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index af9633b7bb6..16aa75c2838 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1465,5 +1465,220 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.14", "user_agent.version": "72.0." + }, + { + "@timestamp": "2022-08-24T21:04:17.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "2a02:cf40:: - remote_monitoring_user [24/Aug/2022:21:04:17 +0000] \"POST /_bulk HTTP/1.1\" 200 470 \"-\" \"Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)\" 2057 0.033 [esmon-esmon-es-http-9200] [] [2a02:cf40::]:9200 470 0.036 200 3db73c6c673c4256ade033a6ce08c2ab", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "3db73c6c673c4256ade033a6ce08c2ab", + "http.request.method": "POST", + "http.response.body.bytes": 470, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 6375, + "nginx.ingress_controller.http.request.id": "3db73c6c673c4256ade033a6ce08c2ab", + "nginx.ingress_controller.http.request.length": 2057, + "nginx.ingress_controller.http.request.time": 0.033, + "nginx.ingress_controller.remote_ip_list": [ + "2a02:cf40::" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "2a02:cf40::", + "nginx.ingress_controller.upstream.name": "esmon-esmon-es-http-9200", + "nginx.ingress_controller.upstream.port": 9200, + "nginx.ingress_controller.upstream.response.length": 470, + "nginx.ingress_controller.upstream.response.length_list": [ + "470" + ], + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], + "nginx.ingress_controller.upstream.response.time": 0.036, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.036" + ], + "nginx.ingress_controller.upstream_address_list": [ + "[2a02:cf40::]:9200" + ], + "related.ip": [ + "2a02:cf40::" + ], + "related.user": [ + "remote_monitoring_user" + ], + "service.type": "nginx", + "source.address": "2a02:cf40::", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "NO", + "source.geo.country_name": "Norway", + "source.geo.location.lat": 62.0, + "source.geo.location.lon": 10.0, + "source.ip": "2a02:cf40::", + "url.original": "/_bulk", + "url.path": "/_bulk", + "user.name": "remote_monitoring_user", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)", + "user_agent.os.name": "Linux" + }, + { + "@timestamp": "2022-08-24T18:05:41.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] \"GET /favicon.ico HTTP/2.0\" 502 552 \"https://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::4e36]:5000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "3db73c6c673c4256ade033a6ce08c2ab", + "http.request.method": "GET", + "http.request.referrer": "https://localhost:8080/", + "http.response.body.bytes": 552, + "http.response.status_code": 502, + "http.version": "2.0", + "input.type": "log", + "log.offset": 6691, + "nginx.ingress_controller.http.request.id": "3db73c6c673c4256ade033a6ce08c2ab", + "nginx.ingress_controller.http.request.length": 27, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "2a02:cf40::4e36" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "2a02:cf40::4e36", + "nginx.ingress_controller.upstream.name": "localhost-8080", + "nginx.ingress_controller.upstream.port": 5000, + "nginx.ingress_controller.upstream.response.length": 0, + "nginx.ingress_controller.upstream.response.length_list": [ + "0", + "0" + ], + "nginx.ingress_controller.upstream.response.status_code": 502, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "502", + "502" + ], + "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000", + "0.000" + ], + "nginx.ingress_controller.upstream_address_list": [ + "[2a02:cf40::4e36]:5000", + "[2a02:cf40::7]:5000" + ], + "related.ip": [ + "2a02:cf40::4e36" + ], + "service.type": "nginx", + "source.address": "2a02:cf40::4e36", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "NO", + "source.geo.country_name": "Norway", + "source.geo.location.lat": 62.0, + "source.geo.location.lon": 10.0, + "source.ip": "2a02:cf40::4e36", + "url.extension": "ico", + "url.original": "/favicon.ico", + "url.path": "/favicon.ico", + "user_agent.device.name": "Mac", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "user_agent.os.full": "Mac OS X 10.15.7", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.15.7", + "user_agent.version": "104.0.0.0" + }, + { + "@timestamp": "2022-08-24T18:05:41.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] \"GET /favicon.ico HTTP/2.0\" 502 552 \"https://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::::::::4e36]:500000000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "3db73c6c673c4256ade033a6ce08c2ab", + "http.request.method": "GET", + "http.request.referrer": "https://localhost:8080/", + "http.response.body.bytes": 552, + "http.response.status_code": 502, + "http.version": "2.0", + "input.type": "log", + "log.offset": 7055, + "nginx.ingress_controller.http.request.id": "3db73c6c673c4256ade033a6ce08c2ab", + "nginx.ingress_controller.http.request.length": 27, + "nginx.ingress_controller.http.request.time": 0.0, + "nginx.ingress_controller.remote_ip_list": [ + "2a02:cf40::4e36" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.name": "localhost-8080", + "nginx.ingress_controller.upstream.response.length": 0, + "nginx.ingress_controller.upstream.response.length_list": [ + "0", + "0" + ], + "nginx.ingress_controller.upstream.response.status_code": 502, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "502", + "502" + ], + "nginx.ingress_controller.upstream.response.time": 0.0, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.000", + "0.000" + ], + "nginx.ingress_controller.upstream_address_list": [ + "[2a02:cf40::7]:5000", + "[2a02:cf40::::::::4e36]:500000000" + ], + "related.ip": [ + "2a02:cf40::4e36" + ], + "service.type": "nginx", + "source.address": "2a02:cf40::4e36", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "NO", + "source.geo.country_name": "Norway", + "source.geo.location.lat": 62.0, + "source.geo.location.lon": 10.0, + "source.ip": "2a02:cf40::4e36", + "url.extension": "ico", + "url.original": "/favicon.ico", + "url.path": "/favicon.ico", + "user_agent.device.name": "Mac", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36", + "user_agent.os.full": "Mac OS X 10.15.7", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.15.7", + "user_agent.version": "104.0.0.0" } ] \ No newline at end of file