From e37d14653a15617e5cf1f6f6865c5e0a57d65c7a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 5 Oct 2020 12:42:17 -0600 Subject: [PATCH 1/3] Add support for additional fields from V2 ALB logs --- filebeat/docs/fields.asciidoc | 40 +++++++++++++++++ .../filebeat/module/aws/elb/_meta/fields.yml | 16 +++++++ .../module/aws/elb/ingest/pipeline.yml | 16 +++++++ .../aws/elb/test/application-lb-http.log | 2 +- .../application-lb-http.log-expected.json | 43 +++++++++++++++++++ x-pack/filebeat/module/aws/fields.go | 2 +- 6 files changed, 117 insertions(+), 2 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index b66c1163367..63b7e6a9348 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1884,6 +1884,46 @@ type: keyword The error reason if the executed action failed. +type: keyword + +-- + +*`aws.elb.target_port`*:: ++ +-- +List of IP addresses and ports for the targets that processed this request. + + +type: keyword + +-- + +*`aws.elb.target_status_code`*:: ++ +-- +List of status codes from the responses of the targets. + + +type: keyword + +-- + +*`aws.elb.classification`*:: ++ +-- +The classification for desync mitigation. + + +type: keyword + +-- + +*`aws.elb.classification_reason`*:: ++ +-- +The classification reason code. + + type: keyword -- diff --git a/x-pack/filebeat/module/aws/elb/_meta/fields.yml b/x-pack/filebeat/module/aws/elb/_meta/fields.yml index 9499f8bbb0e..aac074e9347 100644 --- a/x-pack/filebeat/module/aws/elb/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/elb/_meta/fields.yml @@ -101,3 +101,19 @@ type: keyword description: > The error reason if the executed action failed. + - name: target_port + type: keyword + description: > + List of IP addresses and ports for the targets that processed this request. + - name: target_status_code + type: keyword + description: > + List of status codes from the responses of the targets. + - name: classification + type: keyword + description: > + The classification for desync mitigation. + - name: classification_reason + type: keyword + description: > + The classification reason code. diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index de772ccdf01..c572b789e42 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -32,6 +32,10 @@ processors: \"(?:-|%{DATA:_tmp.actions_executed})\" \"(?:-|%{DATA:aws.elb.redirect_url})\" \"(?:-|%{DATA:aws.elb.error.reason})\" + \"(?:-|%{DATA:_tmp.target_port})\" + \"(?:-|%{DATA:_tmp.target_status_code})\" + \"(?:-|%{DATA:aws.elb.classification})\" + \"(?:-|%{DATA:aws.elb.classification_reason})\" # TCP from Network Load Balancers (v2 Load Balancers) - >- @@ -141,6 +145,18 @@ processors: separator: ',' ignore_missing: true + - split: + field: '_tmp.target_port' + target_field: 'aws.elb.target_port' + separator: ' ' + ignore_missing: true + + - split: + field: '_tmp.target_status_code' + target_field: 'aws.elb.target_status_code' + separator: ' ' + ignore_missing: true + - date: field: '_tmp.timestamp' formats: diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log index 88ea2d75c26..ce972a72967 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log @@ -8,4 +8,4 @@ http 2019-10-11T15:03:49.331902Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.2 http 2019-10-11T15:55:09.308183Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37838 10.0.0.192:80 0.001 0.000 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af" "-" "-" 0 2019-10-11T15:55:09.307000Z "forward" "-" "-" http 2019-10-11T15:55:11.354283Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37850 10.0.1.107:80 0.001 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7d64cabe9955b4df9acc800a" "-" "-" 0 2019-10-11T15:55:11.352000Z "forward" "-" "-" http 2019-10-11T15:55:11.987940Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37856 10.0.0.192:80 0.000 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4" "-" "-" 0 2019-10-11T15:55:11.987000Z "forward" "-" "-" - +http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward" "-" "-" 10.0.0.1:80 200 "-" "-" diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json index 28e1564e928..03e32cd0391 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json @@ -500,5 +500,48 @@ ], "tracing.trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4", "user_agent.original": "curl/7.58.0" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "forward" + ], + "aws.elb.backend.http.response.status_code": 200, + "aws.elb.backend.ip": "10.0.0.1", + "aws.elb.backend.port": "80", + "aws.elb.backend_processing_time.sec": 0.001, + "aws.elb.matched_rule_priority": "0", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.0, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", + "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 34, + "http.request.method": "GET", + "http.request.referrer": "http://www.example.com:80/", + "http.response.body.bytes": 366, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4431, + "service.type": "aws", + "source.ip": "192.168.131.39", + "source.port": "2817", + "tags": [ + "forwarded" + ], + "tracing.trace.id": "Root=1-58337262-36d228ad5d99923122bbe354", + "user_agent.original": "curl/7.46.0" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 352932f1b1c..e8968b65e8e 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzcXN+T4raTf9+/oisvma0Crr6b1NXVXOWq2NnJhctkMzewyd2TI+wGdCMkR5JhyV9/1ZL8AywDM5jd1JeHXQbb0qdb/VstD+EZd7fAtuYNgOVW4C2Mf5++AdAokBm8hTla9gYgQ5Nqnluu5C38xxsAgF9UVgiEhdKwYjITXC5BqKWBhVZrGmb0BmDBUWTm1j0wBMnWWE5HH7vL8RaWWhV5+CUyD31+dMNUI7t5RuFqc4rmNKlQRWY146K6FJuRPofUlp8MF6wQNnFT3MKCCYN7l6Ngm4CVdnjvCMuMsOxBj8FvkoAblDbZoDZcyb07SkqecbdVOju4dgQYfWYrbCIK44NagF0hAfQTE/o1s6MotMKgTniG0nK7i0I7ZHIb2DCKjEaehIEBBa4JSqqkZVwayNAyLgywuSqsw0uzgVq0xpqMf4ESINgVs7BmGbpHNP5ZoLEDYDKD7YqnK0g1unuZMLBFja3hCoPZCCYLsLjOlWZ613rG3TNwM5S4zUptDazUln5tjdkaQM2JSsxGB7fGhKS5GsSD1sXjMtJejsgNfkUChx1hHUve0G59KKkvR9IWjBLKeM3+UhKe0KhCpwgf2RrhZvz08W0JMNdcpjxn4mDNUybEIVsbqNMUjUmecZfwGL6+8Pt5aCCYfPAIt8w4wQGrwPClbEpoN2CDhpQ2IcXAz7YTckwLzwU8WTSxOKCOnVtuVw01MJgWOiYSsC/ipG6VYjjSc602PEMDXHpbQ2ao1uxAY3TcinWpRmYxc6bWrpTB5pSRR7tUqcnc9YIlrLArGiWl0aN3n5aKcxkNQTo2TBQI3IDV9H9gv1LWGUVQ2hk1931LpHYOFrVMgUX1gjJhlOPhHq1+eVmc7fT55ccxZLjhKf47KLtCveUGB947tgW2yVe3ViS1GbNd4D1Pj9zwEobSMM7IW75G2K7Qa1dbdtsc48YUbUO8T0+phO5efZSgLj18CUV96CO8Xic7xwvu7Xx3Vn6O6SKcdm/l5xw1hBfwGYL0BB8THMtxoRmAKdLV0SGZgSel7ICU+JNBPSCFflKiQ2maDKicWtw7XZsRXFrUkgnyWYEbzbiq6cGW2C0nsC97p8mOxxLXpnb89LGkMkjADUtTVUi/dM7+urXTSuDbo8PF2HNCkM7gigfzdUQhTO4pU1tpricNJb1cbtQzZsk8ZtH6CsxoqnLVKWMzqMnDdSUOpOzAYvEFlDHq/d07GBdWwTRlLjkOueC9YMbyFN4jk8Yy8RxPsFBrpZNUZYeW7/zEL55fNalzk1SBRvArGm2hpXGega4fw7dGY9iyT4iT42B8etUYpDJI3VDDWEnONFujRX24bpeytB54QMxkcjcIukBe0JBv9T66O7JfF8Ly5GSedyzU77h4oojRZJPJlTSYhHigby6V41fxBsWjLKVnTKlnzwjpisklGrjxkf2gnYnnFNY5C5yhQIrw/CBv/4ZMZVnGCRwTiSuqZGyvwnQpX8fV8BTrskby5Es4lUWWypKY2qAurYFKfSO3Ftbpb8jNUpNbru9S2fSZl+OWr3AsOJo9ffURYbCbc+Ry2S7nMCEwgyVK1My657nxQ3fYUFffi0S3FxnQffxl8aYhDyXArCEoGlOlszhMlvOLi5AncY4fJ1UlkhmjUl4no+761oxzfseEaI3kKJgRnUd4vWaSLZ3d8YrYpxLCe6UEMtkhRtsVUprc4DY3cGgFoIHQ39XlzFiWKCniRdeLl6LGyg2onOSEVoQAu6mHNHV9oQujD5zjHuQ1heExCG6c8arGDrU0zIDLmrUvrZxer1xZ1SjHTx/bgeJZwXwfMMYhYK9Tt5KDFL5Hqha91pRPsMYZp0pTdLmSvvDg9qJub9nWDIPdHTpkt+TphvSo+7tDAlOec1L2TgZfojBPmGukuM7bLlbz2Om+xhT5xtlXbo4pc6DLW6Qk7G1cz8ZWYT9NNwAuU1FklJpsCbXVfLlE7d1C3Mj6WpqXoUL8HYNYs2Ias8DQXtf8Pz9NPjRc53zX3EOzCgrJ/yxQ7Ep5bl6PczNsaLqVofSTMjMfygYXYnzuYBVkfLFATX/4/dn9T5A/ExeyTZ4mKLNc8b5ZciBevz3eQTkRqbLfWQsBVCgJulTakd12gPS8VcCkqyo3E9Uq4S6T6+l3cVpTJY0SmAi15PFg5TXeJ+zmmhxTvuApgbzzEz3QPGE1X+p5TmcGx1G3kccdQJ0i3Dsd/kCJAuVep2k4RkeTlrWac4EdQeI+JXMfKnXcc3YpvB2x7FFDKY+TMo8MjiEriXAyk1h1FH4/mz2fnh5aK3CcwQuWFObERtTVWOvsOixYailjrzeLyFJtO6pgEPbMISs0uZhOUksSF4JZi7JF4+vV9n5aD+rSYxduEN/V/P8wtY5A7St7pph7QQemUX5r4VmqrSQrxrINk2krG+5Vvbtob1N5SsUvqwLAeZWAF5TWXkNjd5W2h0Lb+fWufqH3Vf2CF1XAXhZq9kdxj4EnnBV8lmRmfIkmntxf4PmdK7mrGsrgg5sFHtTypV5fqGWy4KKVGtcwJZp4x8FZ2V6VJz+opZunbJ2q82TPoiOSYpm2ieXr7vSvY3v+XF1wM9Dyf5rd+a15TVLv1aCGCEQApIqcd1wRLHumYZivTrgIOfX23I1axaBCLcFzY8U2cRGbI0rSJr5xkrgX3nfzCmV2VU6hzP4p+GS+S+ZF+hzdDrzKFl6ZJoCflkJ2T6JrVCi0bpcAw2wNlq6YOaD3KIU+qrgihTVVfirXR3YTcsVBlPDoWEL54K1qmgsMaRLfTanELbn8ULy+ovBXUh9QrpXxtZWyQgpsrUishTgW0IRcO1i/WsTPNIdKZF+HXD9xB6Xd+Rq8mtJc44arwiRfQluPa2gJZU8du1KNM7SzIm3FzCphYqk0t6v1F7JGNClUk7bbJNz1LvGNseIIocVc8NQ1si64XKLONY9aur7oXOFnlmHK10wAylRlmEFj5qot1+FyBovIjw63ZjZdhYgx13zDLLoHDrpj+ZmsoNuZLTR+kfWul7fVzNuJtu5wMXy5Ol4A7orIT+Gbug70eINtITPUYkehQYjBDa0XkzDxiNr6FiL4shznjZOv5g98Zypb+jsss9xYnpqB29sjStvBiW/Aj2yP2TJMpCQHZVbK0XFgNXv3ToFsSbDeHHL065wC+Z2wvOgUyLEem0i5/ES9YB9EOXibcZi++/ocu797588UcdkAfi7jeJ6wLNNoXr+l0mJf3Q6JFsLo9caaqzigjnBTzM/n5vI6vHx4/yKxa+2tXcy3pksUimUwZ4LJFDvati7qi4gCaB5k2QPgmLR5Bw/04/vwY8d2imV6iTZxqzdqbx9fCLHRfOon8mJSH7HrLG1VFQZuyE8cbuxeiMtJTxi5tclJyaeUmHZ3BORaWZWqw42wC0GVo8bX9GZlbU7uw6b52xOdgVqlaAyXSxfhjwymHU5YtWKXc+ROWSaqVNtgqmRmwPAyCa+556uuvgDLTc3jQlougO/tCVIqv6QlcXk4S59RdjTwhIt/IzIbZNCVABAsF2LvBxcCmFA8zbhcdjac+FLuV6awKvI2167a5N2jcn8tHXsE72wUqFnn6VrHXZpQrba0l1FV1gMaKyVhzYXggdhBoNbDV7nbW2kQlAplDhOxynIKyr9kZlbsGa9LR3nUafYwhWpKYnSq1rmrmB/QBSoipVX9B41lc8HNqou0Uv34YYH5Qgs3eTwMMkohqiXdp0SnLHCJMFf69b1vcSustC1LDJeiI5M9qradKIcozGVN8FHIfmCggdvqedO82lZQ/wD5Q9Jfq7SvrP/RCf2PKMXGiCTl+apvRz2dPoAf1yeiXJIS/Iv7uVqEjtCGMF3HU9P0lbd+Ma50pQzKJEVtrxpx+XmA5uELdxoSQp+Xz+gbQvBa+AY1Zz0z148JsljPUV+ZFi5TtXbuVZiECezbmFButUQdOmnVwllwN0/Do853sRC+1GNPldsgDhR3uyP6liWxncEL6SDYbnAfv3cg0CzFS1qyolO7I+HSVh7jj/8Zjtd/yeGMZhtOsj9ghSzryrp8ES5LdCEonOIqcpT04rDdj1ovsoudClEdt3IQ9k9c8QUFvHRPuNzRq+6DJPyMadHeRr0QeGjOKwf3B4r3XF19cOlmofSW6WwAC/4Zs2HpGQZ7p61Ho9HbEUwspEyWO7VgcIOaCc+eDj3UmHGNqU0K3bM1+fT0ECy043iYxxUO07Ljp2LBkbNhI43M9P3aEn9gzY9cnlurliPgWzAumsjqHTrfMf61CzHT71w/BOrybRAvqcr4zZIk1rp9qdlgUkmeMuHLuHW/uJvr4ESsh9ERy8W2jnosHZX7ReUmePM8fFBGEoYl47KrRqJxrSwmHdF66+dzTEOeM+2d6dnFQWhXIr7YolbzuS4iBkMnnoXcfxVE2bt7vHrSpwMbg7GuX3Cv07reLIw2WiOr+4zjUKsDK/2ytz4g42pjGaz8qyMgw1QwygyYgemv48dRdecAnu6ns9FPs9ljska7UtmoPJDhToIN4Pf799PJ7P7YLUrD+/Hs7qfRh/uH+9n96Nf3/3V/N4uT/ow9u+9vnnH3TbONsHbS5DvCJqAD+c3wm9JK16zKFPpOREs5OXNbfFVj33FJKzTvl5YnP/Dw09NkjyLifWVYWnskTWiU9iU+2+uxlCGLNWqeehzNfLQ+wxPpzOzx7Hg8VarU8N754TuVYXOdpQoOWqWuo6SrWLKzaBLTdQjv1RwL2U9VhHPzuAx+APi5bDl0LK0rxxvUFA43yfgLteowI67tJjH8rzhnL6lb0aCVy/XtPVyCk8DOsol7MtaU0g8n9wpkh66WS1gIvlzZxikWF9Z8ayBHbXKKGjcdEmoLLROmVRHvn7sKfGYbAmxyctaN2H2nCn3ch7jGOd23h97LgZxsPoV5Qpp2OpF17+Jz2+7XhvbJoB6OaaajOWQ4bdF7ZlueD558KOuNlec519mEISbZKZezUj1HNkTA5yFb/zXk2fCdextIJY342aLM6oALJh86KnRVT8tV3gpZDV/yaQBTvvzNoaUv3w/azUvNiHHPSLw6rvTly8QUvNVPd2kZElMibqoogzDwwHao4WY6fXhb1kzroxK4VJZXr5Yj8Z/GSKMLHaWIvYMylx3zj/uNsJ1dvc9t/2SOfxvluLCrn5yu+hMH+/d4LTYD+O8C9W7qQ2+670/6u4zFb3KNQ5INzCjEe/v6pXVa5SftuTBQnjssxTJUMunriTODVpjraNNMM2nc7ogXtGn5uqeb2cP0bWXNGpIWCpuHO4GNk5wLobbnFzCu1Zfz2+MdEJIXlS6uwmNC8iMheVBLU07h3tO6UwUJQ3hfkCM8HCz3ndIl+7mBd9UD/ojnDhikhbFq3fVEhyj1cNg8Hni7k8HVIfNyd7Ncgq5SvUW9uEaFuS4jSLRbpZ/ruRy2uo/XarZY8DTshyudHa/b9gvz4Cx17K0iAd8Axnd3948z90a+++5cWqjlsVzv1UiFWi7J0IZMLzC3XN4B/PrzAD7++mE8GztP/PPkkb53NpNaJq+66uUUjrXftjn7CqkYlKFbNTY3rvLojOJOFR19Rc82MTplWRb3J68p5eWMooOhwA0KuFGaL7lk4m1Z+mxvyQdyuhFmxn4RhBnlitJ79gbM0lwcxbnJ0ytKjDvaT3pYvci7V+thirnE/s1ujd9PcE0SbJonC8FaBwovJGHO7ZqZ55DLVY5DCaG2ZHFmd4/gpr2Fdz9M//fj4B//Rv8Nx3c/D/7xw4+Tj4Pvf3iazuKQr9eg6bl2C5PHzfcD+vdfXYp3/+N49Ob/AwAA//+8xDjD" + return "eJzcXN9z47Zzf7+/Yicv8c1I7nwvmU7HnXRG53MaNc7FtXxJ+8RA5IpCDQEMAEqn/PWdBcAfEkFJtqi7zFcPd7JIAp9d7G8sOIZn3N4A25g3AJZbgTcw+X32BkCjQGbwBuZo2RuADE2qeWG5kjfwH28AAH5RWSkQFkrDkslMcJmDULmBhVYrGub6DcCCo8jMjXtgDJKtsJqOPnZb4A3kWpVF+CUyD31+dMPUI7t5rsPV9hTtaVKhysxqxkV9KTYjffaprT4ZLlgpbOKmuIEFEwZ3LkfBtgEr7fDeEpYnwrIDPQa/TQKuUdpkjdpwJXfuqCh5xu1G6Wzv2gFg9HlaYhtRGB/UAuwSCaCfmNCvmL2OQisN6oRnKC232yi0fSZ3gY2jyGjkaRgYUOCKoKRKWsalgQwt48IAm6vSOrw0G6hFZ6zp5BeoAIJdMgsrlqF7ROOfJRo7AiYz2Cx5uoRUo7uXCQMb1NgZrjSYXcN0ARZXhdJMbzvPuHtGboYKt1mqjYGl2tCvnTE7A6g5UYnZ9d6tMSFprwbxoHPxsIx0lyNyg1+RwGFHWM+St7Rb70vqy5F0BaOCMlmxv5SERzSq1CnCR7ZCuJo8fnxbASw0lykvmNhb85QJsc/WFuo0RWOSZ9wmPIZvKPx+HhoIph88wg0zTnDAKjA8l20J7Qds0JDSJqQY+Nn2Qo5p4amAp4s2FgfUsXPD7bKlBgbTUsdEAnZFnNStVgxHeqHVmmdogEtva8gMNZodaIyOW7Mu1cgsZs7U2qUy2J4y8mifKrWZu1qwhJV2SaOkNHr07uNScSqjIUjHmokSgRuwmv4P7FfKOqMISjuj5r5viNTewaKWKbCoWVAmjHI83KHVLy+Ls50+v/w4gQzXPMV/B2WXqDfc4Mh7x67Atvnq1oqkNmO2D7zn6YEbXsJQGsYZectXCJsleu3qym6XY9yYsmuId+mplNDdqw8S1KeHL6FoCH2E1+tk73jBvZ3uzqrPIV2E4+6t+pyihvACPkOQnuBjgmM5LDQjMGW6PDgkM/ColB2REn8yqEek0I9K9ChNmwG1U4t7p0szgkuLWjJBPitwox1XtT1Yjv1yAruyd5zseCxxaWonjx8rKoMEXLE0VaX0S+fsr1s7rQS+PThcjD1HBOkErngwX0cUwuSeMrWR5nLSUNHL5Vo9Y5bMYxZtqMCMpqpWnTI2g5o8XF/iQMoOLBZfQBWj3t2+g0lpFcxS5pLjkAveCWYsT+E9MmksE8/xBAu1VjpJVbZv+U5P/OL5VZs6N0kdaAS/otGWWhrnGej6IXwrNIblQ0KcHgbj06vWILVB6ocaxkoKptkKLer9dTuXpc3AI2Imk9tR0AXygoZ8q/fR/ZH9qhSWJ0fzvEOhfs/FI0WMNptMoaTBJMQDQ3OpGr+ONygeZSk9Yyo9e0ZIl0zmaODKR/ajbiZeUFjnLHCGAinC84O8/RsylWUZJ3BMJK6okrGdCtO5fJ3Uw1Osy1rJky/h1BZZKktiaoO6dAaq9I3cWlinvyE3K03uuL5zZdNnXo5bvsKx4Gh29NVHhMFuzpHLvFvOYUJgBjlK1My657nxQ/fYUFffi0S3ZxnQXfxV8aYlDxXArCUoGlOlszhMVvCzi5BHcU4epnUlkhmjUt4ko+76xkwKfsuE6IzkKHgiOg/wesUky53d8Yo4pBLCe6UEMtkjRpslUprc4jY3sG8FoIXQ39XnzFiWKCniRdezl6LByg2oguSEVoQAu6nHNHVzoQ+jD5zjHuQ1heEJCG6c8arHDrU0zIDLhrUvrZxerlxZ1ygnjx+7geJJwfwQMCYhYG9St4qDFL5HqhaD1pSPsMYZp1pTdLWSvvDg9qJubtjGjIPdHTtkN+TpxvSo+7tHAlNecFL2XgafozCPWGikuM7bLtbw2Om+xhT52tlXbg4pc6DLW6Qk7G1czsbWYT9NNwIuU1FmlJpsCLXVPM9Re7cQN7K+luZlqBR/xyDWLJnGLDB00DX/z0/TDy3XOd+299CsglLyP0sU20qe29fj3Awbmm5lKP2kzMyHssGFGJ87WAUZXyxQ0x9+f3b3E+TPxIVsXaQJyqxQfGiW7InXbw+3UE1Equx31kIAFUqCLpV2ZHcdID1vFTDpqsrtRLVOuKvkevZdnNZUSaMEJkLlPB6svMb7hN1cU2DKFzwlkLd+onuaJ6zmSz3P8czgMOou8rgDaFKEO6fDHyhRoNzrOA2H6GjTslJzLrAnSNylZO5DpZ57Ti6FdyOWHWoo5XFS5pHBIWQVEU5mEqsOwh9ms+fT431nBQ4zeMGS0hzZiLoYa51dhwVLLWXszWYRWapNTxUMwp45ZKUmF9NLakXiQjBrUXZofL3a3s2aQV167MIN4rua/x+m1hGofWXPlHMv6MA0ym8tPEu1kWTFWLZmMu1kw4Oqdx/tXSqPqfh5VQA4rRLwgtLaa2jsr9IOUGg7vd41LPShql/wogrYy0LN4SgeMPCEk4LPisyM52jiyf0Znt+5ktu6oQw+uFngXuUv9fpC5cmCi05q3MCUaOIdBydle3WefK9yN0/VOtXkyZ5FByTFMm0Ty1f96V/P9vypuuBmoOX/9HTrt+Y1Sb1XgwYiEAGQKnLecUWw7JmGYb464SLk1NtzN2odgwqVg+fGkq3jIjZHlKRNfO0kcSe87+cVyuyinEKZ/VPwyXyXzMv0ObodeJEtvCpNAD8theyeRNeoUGrdLQGG2VosXTKzR+9BCn1UcUEKG6r8VK6P7CrkiqMo4dGxhPLBW900FxjSJr6fUokbcvmheH1B4a+lPqBcKeNrK1WFFNhKkVgLcSigCbl2sH6NiJ9oDpXIvg65fuIeSvvzNXg1pYXGNVelSb6Eth7W0ArKjjr2pRonaGdN2pKZZcJErjS3y9UXskY0KdSTdtsk3PU+8Y2x4gCh5Vzw1DWyLrjMUReaRy3dUHQu8TPLMOUrJgBlqjLMoDVz3ZbrcDmDReRHh1sxmy5DxFhovmYW3QN73bH8RFbQ7cyWGr/IejfL22nm7UXbdLgYni8PF4D7IvJj+GauAz3eYFvKDLXYUmgQYnBD68UkTD2irr6FCL4qx3nj5Kv5I9+ZynJ/h2WWG8tTM3J7e0RpNzjxDfiR7TFbhYmU5KDMKjk6DKxh784pkA0J1pt9jn6dUyC/E5YXnQI51GMTKZcfqRfsgqgG7zIO03dfn2N3t+/8mSIuW8BPZRwvEpZlGs3rt1Q67GvaIdFCGL3ZWHMVB9QRbor56dzML8PL+/cvErvO3trZfGu7RKFYBnMmmEyxp23rrL6IKID2QZYdAI5J63dwTz++Dz/2bKdYpnO0iVu96+728ZkQW82nfiIvJs0Ru97SVl1h4Ib8xP7G7pm4nPSEkTubnJR8Solpf0dAoZVVqdrfCDsTVDVqfE2vltYW5D5sWrw90hmoVYrGcJm7CP/aYNrjhFUndjlF7pRlok61DaZKZgYMr5Lwhnu+6uoLsNw0PC6l5QL4zp4gpfI5LYnLw1n6jLKngSdc/BuR2SKDrgSAYLkQOz+4EMCE4mnGZd7bcOJLuV+ZwrrI2167epN3h8rdtXTsEby3UaBhnadrFXdpQnXa0l5GVVUPaK2UhBUXggdiR4FaD18Vbm+lRVAqlNlPxGrLKSj/kplZsme8LB3VUaen+xnUUxKjU7UqXMV8jy5QESmt6z9oLJsLbpZ9pFXqx/cLzGdauOnDfpBRCVEj6T4lOmaBK4SF0q/vfYtbYaVtVWI4Fx2Z7Ot624lyiNKc1wQfhewHBhq4q55X7atdBfUPkD8k/bVK+8r6H73Q/4hSbIxIUl4sh3bUs9k9+HF9IsolKcG/uJ/rRegJbQjTZTw1TV976xfjSpfKoExS1PaiEZefB2gevnCnISH0efmMviUEr4VvUHM2MHP9mCDL1Rz1hWnhMlUr516FSZjAoY0J5VY56tBJqxbOgrt5Wh51vo2F8JUee6rcBnGguN8d0bcsie0MnkkHwXaD+/i9B4FmKZ7TkhWd2h0Jl7b2GH/8z3iy+kuOn2i28TT7A5bIsr6syxfhskSXgsIpriJHSc8O2/2ozSK72KkU9XErB2H3xBVfUMBL94TLPb3qPkjCz5iW3W3UM4GH5rxqcH+geMfVNQeXrhZKb5jORrDgnzEbV55htHPa+vr6+u01TC2kTFY7tWBwjZoJz54ePdSYcY2pTUo9sDX59HgfLLTjeJjHFQ7TquOnZsGBs2HXGpkZ+rUl/sCaH7k6t1YvR8C3YFz0hqA+eR82CLoPW+5NsIbGVTZpGlPHon5uUx+zDt3rLj46mNEH0BeJhirsrZin1ZZaCa3ZrUn0+TnBjPHOZvD31eyO7XiaodnKFFbc8vzAYYTdJ5NLSOUeuCCexMtuJdB8588tfO1y4Ow715WDunonyUtqg37LLokdIDiXlUwqyVMm/GZCc2rBzbV3LtvD6MkoYhuYAxYwq13LqhWj/VaGRrlZzrjs02uNK2Ux6ckZOz+f4qCKgmkf0p1cooZuPeyLLWo9n+tlYzB24lnK3ReSVB3kh2t4Q4ZREzDWda3u9Ps3W9bRdn9kTbd7HGp9bGpY9jbHtFyFNoOlf4EJZJgKRvkpMzD7dfJwXd85gse72dP1T09PD8kK7VJl19WxIHcecQS/372fTZ/uDt2iNLyfPN3+dP3h7v7u6e761/f/dXf7FCf9GQcOIr95xu037WbWJlSkCCZsRTuQ34y/qWKFhlWZQt8Pa9kzAnMbzXV76WFJKzUflpZHP/D40+N0hyLifW1YOjt1bWhLa4sQIAxYUJPlCjVPPY52VaQ5SRbpDx7wDQbxhL1WwzsXDd6qDNvrLFUIE1Xq+pr6SnZbiyYxfUdBX82xkIPXpWA3j6sjjQA/V42vjqXN/sUaNSVlbTL+Qq16zIhr/koM/yvO2XOqpzRo7XJ9kxmX4CSwN8JyT8Zao4bh5E6Zdt/VcgkLwfOlbZ2lcmHNtwYK1Kag3GXdI6G21DJhWpXxLs6LwGe2JcCmIGfdyiC3qtSHfYhr39RDe+idTNzJ5mOYJxQLjpdT3BshXfPHpaF9MqjHE5rpYCUjnPkZvL5SnVKffqiq3rXnOdXZhCGm2TGXs1QDRzZEwOcxW/015tn4nXsnTS2N+NmizJqAC6YfeurEdWfVRd5NWg9f8WkEM57/5tDSl+9H3Ra6dsS4YyReHVf6InpiSt7p6jy3GI4pETdTlEEYuGdb1HA1m92/rSr3zYEdzJXl9QsOSfxnMdLoQk9BbOe41nkvm4j7jdBUUb9VcPd8mH8n6qS0y5+crvpzL7v3eC02I/jvEvV25kNvuu9P+ruKxa8KjWOSDcwoxHv7+qV1WuUnHbg8VZ1+rcQy1NPp65GTq1aYy2jTk2bSuD06L2iz6qVjV0/3s7e1NWtJWiiv7+9Ht84TL4TanF7AuFR32G8Pt0BIXlS6uAiPCcmPhORe5aaawr0teKtKEobw1ipHeHi9ge/Xr9jPDbyrH/AHjbfAIC2NVau+J3pEaYBXHsQDb3c+vX7VQVXXrJagb8PIol5cYp+jKSNItBuln5u5HLamm9xqtljwNHRlKJ0d3j0YFubeif7Yu20CvhFMbm/vHp7ceyHv+nNpofJDud6rkQqV52RoQ6YXmFst7wh+/XkEH3/9MHmaOE/88/SBvve2NFsmL7rq1RSOtd92OfsKqRhVoVs9Njeu8uiM4laVPd1tzzYxOmVZFvcnrynlFYyig7HANQq4UprnXDLxtip9dhtDAjn9CDNjvwjCjHJF6T17C2a9DXII57pILygx7gUTpIf16+QHtR6mnEsc3uw2+P0ElyTBpkWyEKxzrPVMEubcrph5Drlc7TiUEGpDFufp9gHctDfw7ofZ/34c/ePf6L/x5Pbn0T9++HH6cfT9D4+zpzjky7UJe67dwPRh/f2I/v1Xl+Ld/Ti5fvP/AQAA//9d8O3V" } From 0ca82ca0cd818626fd4d9aa52811ce1f2758ab86 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 5 Oct 2020 15:59:59 -0600 Subject: [PATCH 2/3] Add new fields as optional fields and regenerate -expected.json files --- .../module/aws/elb/ingest/pipeline.yml | 6 +- .../aws/elb/test/application-lb-http.log | 2 +- .../application-lb-http.log-expected.json | 9 +- .../module/aws/elb/test/example-alb-http.log | 5 +- .../test/example-alb-http.log-expected.json | 215 ++++++++++++++++++ 5 files changed, 229 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index c572b789e42..8cb2a914921 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -31,11 +31,7 @@ processors: %{TIMESTAMP_ISO8601:event.start} \"(?:-|%{DATA:_tmp.actions_executed})\" \"(?:-|%{DATA:aws.elb.redirect_url})\" - \"(?:-|%{DATA:aws.elb.error.reason})\" - \"(?:-|%{DATA:_tmp.target_port})\" - \"(?:-|%{DATA:_tmp.target_status_code})\" - \"(?:-|%{DATA:aws.elb.classification})\" - \"(?:-|%{DATA:aws.elb.classification_reason})\" + \"(?:-|%{DATA:aws.elb.error.reason})\"( \"(?:-|%{DATA:_tmp.target_port})\")?( \"(?:-|%{DATA:_tmp.target_status_code})\")?( \"(?:-|%{DATA:aws.elb.classification})\")?( \"(?:-|%{DATA:aws.elb.classification_reason})\")? # TCP from Network Load Balancers (v2 Load Balancers) - >- diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log index ce972a72967..5d754c4bbaa 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log @@ -8,4 +8,4 @@ http 2019-10-11T15:03:49.331902Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.2 http 2019-10-11T15:55:09.308183Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37838 10.0.0.192:80 0.001 0.000 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af" "-" "-" 0 2019-10-11T15:55:09.307000Z "forward" "-" "-" http 2019-10-11T15:55:11.354283Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37850 10.0.1.107:80 0.001 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7d64cabe9955b4df9acc800a" "-" "-" 0 2019-10-11T15:55:11.352000Z "forward" "-" "-" http 2019-10-11T15:55:11.987940Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37856 10.0.0.192:80 0.000 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4" "-" "-" 0 2019-10-11T15:55:11.987000Z "forward" "-" "-" -http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward" "-" "-" 10.0.0.1:80 200 "-" "-" +http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward,redirect" "-" "-" "10.0.0.1:80" "200" "-" "-" diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json index 03e32cd0391..3682fb6520e 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json @@ -504,7 +504,8 @@ { "@timestamp": "2018-07-02T22:23:00.186Z", "aws.elb.action_executed": [ - "forward" + "forward", + "redirect" ], "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.1", @@ -516,6 +517,12 @@ "aws.elb.request_processing_time.sec": 0.0, "aws.elb.response_processing_time.sec": 0.0, "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.1:80" + ], + "aws.elb.target_status_code": [ + "200" + ], "aws.elb.trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", "aws.elb.type": "http", "cloud.provider": "aws", diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log index 9e4526d2d61..94c0ec1360b 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log @@ -7,4 +7,7 @@ http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.13 http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 502 - 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "LambdaInvalidResponse" http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - -1 -1 -1 400 - 0 0 "- http://www.example.com:80- -" "-" - - - "-" "-" "-" 0 2018-11-30T22:22:48.364000Z "-" "-" "-" http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - -1 -1 -1 400 - 0 0 "- - -" "-" - - - "-" "-" "-" 0 2018-11-30T22:22:48.364000Z "-" "-" "-" - +h2 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.1.252:48160 10.0.0.66:9000 0.000 0.002 0.000 200 200 5 257 "GET https://10.0.2.105:773/ HTTP/2.0" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337327-72bd00b0343d75b906739c42" "-" "-" 1 2018-07-02T22:22:48.364000Z "redirect" "https://example.com:80/" "-" "10.0.0.66:9000" "200" "-" "-" +https 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" "www.example.com" "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012" 1 2018-07-02T22:22:48.364000Z "authenticate,forward" "-" "-" "10.0.0.1:80" "200" "-" "-" +ws 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:40914 10.0.1.192:8010 0.001 0.003 0.000 101 101 218 587 "GET http://10.0.0.30:80/ HTTP/1.1" "-" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 1 2018-07-02T22:22:48.364000Z "forward" "-" "-" "10.0.1.192:8010" "101" "-" "-" +wss 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:44244 10.0.0.171:8010 0.000 0.001 0.000 101 101 218 786 "GET https://10.0.0.30:443/ HTTP/1.1" "-" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 1 2018-07-02T22:22:48.364000Z "forward" "-" "-" "10.0.0.171:8010" "101" "-" "-" diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json index eb1fad5f705..2c1490142fa 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json @@ -368,5 +368,220 @@ ], "tracing.trace.id": "-", "user_agent.original": "-" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "redirect" + ], + "aws.elb.backend.http.response.status_code": 200, + "aws.elb.backend.ip": "10.0.0.66", + "aws.elb.backend.port": "9000", + "aws.elb.backend_processing_time.sec": 0.002, + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.redirect_url": "https://example.com:80/", + "aws.elb.request_processing_time.sec": 0.0, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.elb.ssl_protocol": "TLSv1.2", + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.66:9000" + ], + "aws.elb.target_status_code": [ + "200" + ], + "aws.elb.trace_id": "Root=1-58337327-72bd00b0343d75b906739c42", + "aws.elb.type": "h2", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 5, + "http.request.method": "GET", + "http.request.referrer": "https://10.0.2.105:773/", + "http.response.body.bytes": 257, + "http.response.status_code": 200, + "http.version": "2.0", + "input.type": "log", + "log.offset": 3284, + "service.type": "aws", + "source.ip": "10.0.1.252", + "source.port": "48160", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337327-72bd00b0343d75b906739c42", + "user_agent.original": "curl/7.46.0" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "authenticate", + "forward" + ], + "aws.elb.backend.http.response.status_code": 200, + "aws.elb.backend.ip": "10.0.0.1", + "aws.elb.backend.port": "80", + "aws.elb.backend_processing_time.sec": 0.048, + "aws.elb.chosen_cert.arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012", + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.086, + "aws.elb.response_processing_time.sec": 0.037, + "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.elb.ssl_protocol": "TLSv1.2", + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.1:80" + ], + "aws.elb.target_status_code": [ + "200" + ], + "aws.elb.trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259", + "aws.elb.type": "https", + "cloud.provider": "aws", + "destination.domain": "www.example.com", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 0, + "http.request.method": "GET", + "http.request.referrer": "https://www.example.com:443/", + "http.response.body.bytes": 57, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 3750, + "service.type": "aws", + "source.ip": "192.168.131.39", + "source.port": "2817", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337281-1d84f3d73c47ec4e58577259", + "user_agent.original": "curl/7.46.0" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "forward" + ], + "aws.elb.backend.http.response.status_code": 101, + "aws.elb.backend.ip": "10.0.1.192", + "aws.elb.backend.port": "8010", + "aws.elb.backend_processing_time.sec": 0.003, + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.001, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.1.192:8010" + ], + "aws.elb.target_status_code": [ + "101" + ], + "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "aws.elb.type": "ws", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 218, + "http.request.method": "GET", + "http.request.referrer": "http://10.0.0.30:80/", + "http.response.body.bytes": 587, + "http.response.status_code": 101, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4306, + "service.type": "aws", + "source.ip": "10.0.0.140", + "source.port": "40914", + "tags": [ + "forwarded" + ], + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "user_agent.original": "-" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "forward" + ], + "aws.elb.backend.http.response.status_code": 101, + "aws.elb.backend.ip": "10.0.0.171", + "aws.elb.backend.port": "8010", + "aws.elb.backend_processing_time.sec": 0.001, + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.0, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.elb.ssl_protocol": "TLSv1.2", + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.171:8010" + ], + "aws.elb.target_status_code": [ + "101" + ], + "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "aws.elb.type": "wss", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 218, + "http.request.method": "GET", + "http.request.referrer": "https://10.0.0.30:443/", + "http.response.body.bytes": 786, + "http.response.status_code": 101, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4708, + "service.type": "aws", + "source.ip": "10.0.0.140", + "source.port": "44244", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "user_agent.original": "-" } ] \ No newline at end of file From e20f7ce3d6eeab15945be238ce5032bc91b9d010 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 5 Oct 2020 16:14:05 -0600 Subject: [PATCH 3/3] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 279eda229a7..99115ef4e52 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -605,6 +605,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add related.hosts ecs field to all modules {pull}21160[21160] - Keep cursor state between httpjson input restarts {pull}20751[20751] - Convert aws s3 to v2 input {pull}20005[20005] +- Add support for additional fields from V2 ALB logs. {pull}21540[21540] *Heartbeat*