Skip to content

Commit

Permalink
handling IPv6 within square brackets (backport #32989) (#33008)
Browse files Browse the repository at this point in the history 
* handling IPv6 within square brackets (#32989)
  • Loading branch information
@mergify
mergify[bot] authored Sep 7, 2022
1 parent 4678fc6 commit 8927920
Show file tree
Hide file tree
Showing 4 changed files with 253 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
*Filebeat*

- Fix a hang on `apt-get update` stage in packaging. {pull}32580[32580]
- Fix how to handle IPv6 addresses in the fileset `nginx/ingress_controller` for Filebeat. {pull}32989[32989]

*Heartbeat*

Expand Down
43 changes: 34 additions & 9 deletions filebeat/module/nginx/ingress_controller/ingest/pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ processors:
UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))*
UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))*
UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))*
IP: (?:\[?%{IPV6}\]?|%{IPV4})
ignore_missing: true
- grok:
field: nginx.ingress_controller.info
Expand Down Expand Up @@ -139,18 +140,42 @@ processors:
for (def item : ctx.nginx.ingress_controller.upstream_address_list) {
last_upstream = item;
}
StringTokenizer tok = new StringTokenizer(last_upstream, ":");
if (tok.countTokens()>1) {
ctx.nginx.ingress_controller.upstream.ip = tok.nextToken();
ctx.nginx.ingress_controller.upstream.port = Integer.parseInt(tok.nextToken());
} else {
ctx.nginx.ingress_controller.upstream.ip = last_upstream;
}
ctx.nginx.ingress_controller.upstream.address = last_upstream;
}
catch (Exception e) {
ctx.nginx.ingress_controller.upstream.ip = null;
ctx.nginx.ingress_controller.upstream.port = null;
ctx.nginx.ingress_controller.upstream.address = null;
}
- grok:
field: nginx.ingress_controller.upstream.address
patterns:
- "^%{IPV4:nginx.ingress_controller.upstream.ip}:%{NUMBER:nginx.ingress_controller.upstream.port}$"
- "^\\[%{IPV6:nginx.ingress_controller.upstream.ip}\\]:%{NUMBER:nginx.ingress_controller.upstream.port}$"
- "^%{IPV6NOCOMPRESS:nginx.ingress_controller.upstream.ip}:%{NUMBER:nginx.ingress_controller.upstream.port}$"
- "^%{IPV6:nginx.ingress_controller.upstream.ip}%{IPV6PORTSEP}%{NUMBER:nginx.ingress_controller.upstream.port}$"
- "^%{IPV6:nginx.ingress_controller.upstream.ip}%{IPV6PORTSEP}%{POSINT:nginx.ingress_controller.upstream.port}$"
pattern_definitions:
IPV6NOCOMPRESS: '([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}'
IPV6PORTSEP: '(?: port |[p#.])'
ignore_missing: true
ignore_failure: true
- convert:
field: nginx.ingress_controller.upstream.ip
type: ip
ignore_missing: true
on_failure:
- remove:
field: nginx.ingress_controller.upstream.ip
- convert:
field: nginx.ingress_controller.upstream.port
type: long
ignore_missing: true
on_failure:
- remove:
field: nginx.ingress_controller.upstream.port
- remove:
field: nginx.ingress_controller.upstream.address
ignore_failure: true
- script:
if: ctx.nginx?.ingress_controller?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0
lang: painless
Expand Down
3 changes: 3 additions & 0 deletions filebeat/module/nginx/ingress_controller/test/test.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,6 @@
192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402
192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402
192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402
2a02:cf40:: - remote_monitoring_user [24/Aug/2022:21:04:17 +0000] "POST /_bulk HTTP/1.1" 200 470 "-" "Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)" 2057 0.033 [esmon-esmon-es-http-9200] [] [2a02:cf40::]:9200 470 0.036 200 3db73c6c673c4256ade033a6ce08c2ab
2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::4e36]:5000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab
2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::::::::4e36]:500000000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab
215 changes: 215 additions & 0 deletions filebeat/module/nginx/ingress_controller/test/test.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1465,5 +1465,220 @@
"user_agent.os.name": "Mac OS X",
"user_agent.os.version": "10.14",
"user_agent.version": "72.0."
},
{
"@timestamp": "2022-08-24T21:04:17.000Z",
"event.category": [
"web"
],
"event.dataset": "nginx.ingress_controller",
"event.kind": "event",
"event.module": "nginx",
"event.original": "2a02:cf40:: - remote_monitoring_user [24/Aug/2022:21:04:17 +0000] \"POST /_bulk HTTP/1.1\" 200 470 \"-\" \"Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)\" 2057 0.033 [esmon-esmon-es-http-9200] [] [2a02:cf40::]:9200 470 0.036 200 3db73c6c673c4256ade033a6ce08c2ab",
"event.outcome": "success",
"event.timezone": "-02:00",
"event.type": [
"info"
],
"fileset.name": "ingress_controller",
"http.request.id": "3db73c6c673c4256ade033a6ce08c2ab",
"http.request.method": "POST",
"http.response.body.bytes": 470,
"http.response.status_code": 200,
"http.version": "1.1",
"input.type": "log",
"log.offset": 6375,
"nginx.ingress_controller.http.request.id": "3db73c6c673c4256ade033a6ce08c2ab",
"nginx.ingress_controller.http.request.length": 2057,
"nginx.ingress_controller.http.request.time": 0.033,
"nginx.ingress_controller.remote_ip_list": [
"2a02:cf40::"
],
"nginx.ingress_controller.upstream.alternative_name": "",
"nginx.ingress_controller.upstream.ip": "2a02:cf40::",
"nginx.ingress_controller.upstream.name": "esmon-esmon-es-http-9200",
"nginx.ingress_controller.upstream.port": 9200,
"nginx.ingress_controller.upstream.response.length": 470,
"nginx.ingress_controller.upstream.response.length_list": [
"470"
],
"nginx.ingress_controller.upstream.response.status_code": 200,
"nginx.ingress_controller.upstream.response.status_code_list": [
"200"
],
"nginx.ingress_controller.upstream.response.time": 0.036,
"nginx.ingress_controller.upstream.response.time_list": [
"0.036"
],
"nginx.ingress_controller.upstream_address_list": [
"[2a02:cf40::]:9200"
],
"related.ip": [
"2a02:cf40::"
],
"related.user": [
"remote_monitoring_user"
],
"service.type": "nginx",
"source.address": "2a02:cf40::",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "NO",
"source.geo.country_name": "Norway",
"source.geo.location.lat": 62.0,
"source.geo.location.lon": 10.0,
"source.ip": "2a02:cf40::",
"url.original": "/_bulk",
"url.path": "/_bulk",
"user.name": "remote_monitoring_user",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)",
"user_agent.os.name": "Linux"
},
{
"@timestamp": "2022-08-24T18:05:41.000Z",
"event.category": [
"web"
],
"event.dataset": "nginx.ingress_controller",
"event.kind": "event",
"event.module": "nginx",
"event.original": "2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] \"GET /favicon.ico HTTP/2.0\" 502 552 \"https://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::4e36]:5000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab",
"event.outcome": "failure",
"event.timezone": "-02:00",
"event.type": [
"info"
],
"fileset.name": "ingress_controller",
"http.request.id": "3db73c6c673c4256ade033a6ce08c2ab",
"http.request.method": "GET",
"http.request.referrer": "https://localhost:8080/",
"http.response.body.bytes": 552,
"http.response.status_code": 502,
"http.version": "2.0",
"input.type": "log",
"log.offset": 6691,
"nginx.ingress_controller.http.request.id": "3db73c6c673c4256ade033a6ce08c2ab",
"nginx.ingress_controller.http.request.length": 27,
"nginx.ingress_controller.http.request.time": 0.0,
"nginx.ingress_controller.remote_ip_list": [
"2a02:cf40::4e36"
],
"nginx.ingress_controller.upstream.alternative_name": "",
"nginx.ingress_controller.upstream.ip": "2a02:cf40::4e36",
"nginx.ingress_controller.upstream.name": "localhost-8080",
"nginx.ingress_controller.upstream.port": 5000,
"nginx.ingress_controller.upstream.response.length": 0,
"nginx.ingress_controller.upstream.response.length_list": [
"0",
"0"
],
"nginx.ingress_controller.upstream.response.status_code": 502,
"nginx.ingress_controller.upstream.response.status_code_list": [
"502",
"502"
],
"nginx.ingress_controller.upstream.response.time": 0.0,
"nginx.ingress_controller.upstream.response.time_list": [
"0.000",
"0.000"
],
"nginx.ingress_controller.upstream_address_list": [
"[2a02:cf40::4e36]:5000",
"[2a02:cf40::7]:5000"
],
"related.ip": [
"2a02:cf40::4e36"
],
"service.type": "nginx",
"source.address": "2a02:cf40::4e36",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "NO",
"source.geo.country_name": "Norway",
"source.geo.location.lat": 62.0,
"source.geo.location.lon": 10.0,
"source.ip": "2a02:cf40::4e36",
"url.extension": "ico",
"url.original": "/favicon.ico",
"url.path": "/favicon.ico",
"user_agent.device.name": "Mac",
"user_agent.name": "Chrome",
"user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"user_agent.os.full": "Mac OS X 10.15.7",
"user_agent.os.name": "Mac OS X",
"user_agent.os.version": "10.15.7",
"user_agent.version": "104.0.0.0"
},
{
"@timestamp": "2022-08-24T18:05:41.000Z",
"event.category": [
"web"
],
"event.dataset": "nginx.ingress_controller",
"event.kind": "event",
"event.module": "nginx",
"event.original": "2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] \"GET /favicon.ico HTTP/2.0\" 502 552 \"https://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::::::::4e36]:500000000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab",
"event.outcome": "failure",
"event.timezone": "-02:00",
"event.type": [
"info"
],
"fileset.name": "ingress_controller",
"http.request.id": "3db73c6c673c4256ade033a6ce08c2ab",
"http.request.method": "GET",
"http.request.referrer": "https://localhost:8080/",
"http.response.body.bytes": 552,
"http.response.status_code": 502,
"http.version": "2.0",
"input.type": "log",
"log.offset": 7055,
"nginx.ingress_controller.http.request.id": "3db73c6c673c4256ade033a6ce08c2ab",
"nginx.ingress_controller.http.request.length": 27,
"nginx.ingress_controller.http.request.time": 0.0,
"nginx.ingress_controller.remote_ip_list": [
"2a02:cf40::4e36"
],
"nginx.ingress_controller.upstream.alternative_name": "",
"nginx.ingress_controller.upstream.name": "localhost-8080",
"nginx.ingress_controller.upstream.response.length": 0,
"nginx.ingress_controller.upstream.response.length_list": [
"0",
"0"
],
"nginx.ingress_controller.upstream.response.status_code": 502,
"nginx.ingress_controller.upstream.response.status_code_list": [
"502",
"502"
],
"nginx.ingress_controller.upstream.response.time": 0.0,
"nginx.ingress_controller.upstream.response.time_list": [
"0.000",
"0.000"
],
"nginx.ingress_controller.upstream_address_list": [
"[2a02:cf40::7]:5000",
"[2a02:cf40::::::::4e36]:500000000"
],
"related.ip": [
"2a02:cf40::4e36"
],
"service.type": "nginx",
"source.address": "2a02:cf40::4e36",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "NO",
"source.geo.country_name": "Norway",
"source.geo.location.lat": 62.0,
"source.geo.location.lon": 10.0,
"source.ip": "2a02:cf40::4e36",
"url.extension": "ico",
"url.original": "/favicon.ico",
"url.path": "/favicon.ico",
"user_agent.device.name": "Mac",
"user_agent.name": "Chrome",
"user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"user_agent.os.full": "Mac OS X 10.15.7",
"user_agent.os.name": "Mac OS X",
"user_agent.os.version": "10.15.7",
"user_agent.version": "104.0.0.0"
}
]

0 comments on commit 8927920

Please sign in to comment.