From cae646a5bb8cb220816fe9c948a6f98646756c94 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 12 Oct 2020 12:20:07 -0600 Subject: [PATCH] Add support for additional fields from V2 ALB logs (#304) --- go.mod | 2 +- go.sum | 4 +- .../elb_logs/_dev/test/pipeline/test-alb.log | 1 + .../test/pipeline/test-alb.log-expected.json | 85 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 14 ++- .../data_stream/elb_logs/fields/fields.yml | 20 +++++ packages/aws/docs/README.md | 4 + packages/aws/manifest.yml | 2 +- .../pipeline/test-error-raw.log-expected.json | 6 -- .../test-ingest-raw.log-expected.json | 6 -- 10 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log create mode 100644 packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json diff --git a/go.mod b/go.mod index 05ee2747054..2ca61ebdf2c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/blang/semver v3.5.1+incompatible - github.com/elastic/elastic-package v0.0.0-20201001110805-0bb695cf2b70 + github.com/elastic/elastic-package v0.0.0-20201012164813-861bb9387609 github.com/elastic/package-registry v0.12.0 github.com/magefile/mage v1.10.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 2d1d4637f00..c8a8e120780 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/elastic/elastic-package v0.0.0-20201001110805-0bb695cf2b70 h1:GV6BO1olp6KHgWnL2KOElxTSXxi5zfaiKcYAqZPZmJE= -github.com/elastic/elastic-package v0.0.0-20201001110805-0bb695cf2b70/go.mod h1:u7Hvc2PyfZBOfidOA5JuC4HOeBd7Ms4Ox1fQ+Wa/CRQ= +github.com/elastic/elastic-package v0.0.0-20201012164813-861bb9387609 h1:/qKEFsMwebx9USAUSl6frxWKOWEPYzLYVc1zMmy+UyE= +github.com/elastic/elastic-package v0.0.0-20201012164813-861bb9387609/go.mod h1:u7Hvc2PyfZBOfidOA5JuC4HOeBd7Ms4Ox1fQ+Wa/CRQ= github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M= github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6 h1:Ehbr7du4rSSEypR8zePr0XRbMhO4PJgcHC9f8fDbgAg= diff --git a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log new file mode 100644 index 00000000000..dcb5b85631a --- /dev/null +++ b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log @@ -0,0 +1 @@ +http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward,redirect" "-" "-" "10.0.0.1:80" "200" "-" "-" \ No newline at end of file diff --git a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json new file mode 100644 index 00000000000..69935eea780 --- /dev/null +++ b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json @@ -0,0 +1,85 @@ +{ + "expected": [ + { + "cloud": { + "provider": "aws" + }, + "tracing": { + "trace": { + "id": "Root=1-58337262-36d228ad5d99923122bbe354" + } + }, + "@timestamp": "2018-07-02T22:23:00.186Z", + "http": { + "request": { + "method": "get", + "body": { + "bytes": 34 + }, + "referrer": "http://www.example.com:80/" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 366 + }, + "status_code": 200 + } + }, + "source": { + "port": "2817", + "ip": "192.168.131.39" + }, + "aws": { + "elb": { + "trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", + "matched_rule_priority": "0", + "type": "http", + "request_processing_time": { + "sec": 0.0 + }, + "response_processing_time": { + "sec": 0.0 + }, + "target_port": [ + "10.0.0.1:80" + ], + "protocol": "http", + "target_status_code": [ + "200" + ], + "name": "app/my-loadbalancer/50dc6c495c0c9188", + "backend": { + "port": "80", + "http": { + "response": { + "status_code": 200 + } + }, + "ip": "10.0.0.1" + }, + "target_group": { + "arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067" + }, + "backend_processing_time": { + "sec": 0.001 + }, + "action_executed": [ + "forward", + "redirect" + ] + } + }, + "event": { + "start": "2018-07-02T22:22:48.364000Z", + "end": "2018-07-02T22:23:00.186Z", + "category": "web", + "kind": "event", + "outcome": "success" + }, + "user_agent": { + "original": "curl/7.46.0" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml index 0c31f7256ff..b7edf61adac 100644 --- a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml @@ -29,7 +29,7 @@ processors: %{TIMESTAMP_ISO8601:event.start} \"(?:-|%{DATA:_tmp.actions_executed})\" \"(?:-|%{DATA:aws.elb.redirect_url})\" - \"(?:-|%{DATA:aws.elb.error.reason})\" + \"(?:-|%{DATA:aws.elb.error.reason})\"( \"(?:-|%{DATA:_tmp.target_port})\")?( \"(?:-|%{DATA:_tmp.target_status_code})\")?( \"(?:-|%{DATA:aws.elb.classification})\")?( \"(?:-|%{DATA:aws.elb.classification_reason})\")? # TCP from Network Load Balancers (v2 Load Balancers) - >- @@ -143,6 +143,18 @@ processors: separator: ',' ignore_missing: true + - split: + field: '_tmp.target_port' + target_field: 'aws.elb.target_port' + separator: ' ' + ignore_missing: true + + - split: + field: '_tmp.target_status_code' + target_field: 'aws.elb.target_status_code' + separator: ' ' + ignore_missing: true + - date: field: '_tmp.timestamp' formats: diff --git a/packages/aws/data_stream/elb_logs/fields/fields.yml b/packages/aws/data_stream/elb_logs/fields/fields.yml index 7e60f60dcfb..5c2b658e209 100644 --- a/packages/aws/data_stream/elb_logs/fields/fields.yml +++ b/packages/aws/data_stream/elb_logs/fields/fields.yml @@ -97,6 +97,26 @@ type: keyword description: | The error reason if the executed action failed. + - name: target_port + type: keyword + description: > + List of IP addresses and ports for the targets that processed this request. + + - name: target_status_code + type: keyword + description: > + List of status codes from the responses of the targets. + + - name: classification + type: keyword + description: > + The classification for desync mitigation. + + - name: classification_reason + type: keyword + description: > + The classification reason code. + - name: destination.domain type: keyword description: Destination domain. diff --git a/packages/aws/docs/README.md b/packages/aws/docs/README.md index 5a37d44a60e..46c6e04719a 100644 --- a/packages/aws/docs/README.md +++ b/packages/aws/docs/README.md @@ -235,6 +235,8 @@ For network load balancer, please follow [enable access log for network load bal | aws.elb.backend_processing_time.sec | The total time in seconds since the connection is sent to the backend till the backend starts responding. | float | | aws.elb.chosen_cert.arn | The ARN of the chosen certificate presented to the client in TLS/SSL connections. | keyword | | aws.elb.chosen_cert.serial | The serial number of the chosen certificate presented to the client in TLS/SSL connections. | keyword | +| aws.elb.classification | The classification for desync mitigation. | keyword | +| aws.elb.classification_reason | The classification reason code. | keyword | | aws.elb.connection_time.ms | The total time of the connection in milliseconds, since it is opened till it is closed. | long | | aws.elb.error.reason | The error reason if the executed action failed. | keyword | | aws.elb.incoming_tls_alert | The integer value of TLS alerts received by the load balancer from the client, if present. | keyword | @@ -248,6 +250,8 @@ For network load balancer, please follow [enable access log for network load bal | aws.elb.ssl_cipher | The SSL cipher used in TLS/SSL connections. | keyword | | aws.elb.ssl_protocol | The SSL protocol used in TLS/SSL connections. | keyword | | aws.elb.target_group.arn | The ARN of the target group handling the request. | keyword | +| aws.elb.target_port | List of IP addresses and ports for the targets that processed this request. | keyword | +| aws.elb.target_status_code | List of status codes from the responses of the targets. | keyword | | aws.elb.tls_handshake_time.ms | The total time for the TLS handshake to complete in milliseconds once the connection has been established. | long | | aws.elb.tls_named_group | The TLS named group. | keyword | | aws.elb.trace_id | The contents of the `X-Amzn-Trace-Id` header. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index f5dd2d2c88b..ab369942b9b 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 0.3.8 +version: 0.3.9 license: basic description: AWS Integration type: integration diff --git a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json b/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json index 200ae637b9d..44d9fdc6c8e 100644 --- a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json +++ b/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json @@ -95,12 +95,6 @@ "error": { "message": "field [@timestamp] doesn't exist" } - }, - { - "message": "", - "error": { - "message": "Provided Grok expressions do not match field value: []" - } } ] } \ No newline at end of file diff --git a/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json b/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json index 4f5a6db7a39..01e2f16b30b 100644 --- a/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json +++ b/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json @@ -1128,12 +1128,6 @@ "url": { "original": "/v2/some" } - }, - { - "message": "", - "error": { - "message": "Provided Grok expressions do not match field value: []" - } } ] } \ No newline at end of file