elastic · adriansr · Feb 9, 2021 · Feb 9, 2021 · Feb 9, 2021 · Feb 9, 2021
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -379,6 +379,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
 - Fix goroutines leak with some inputs in autodiscover. {pull}23722[23722]
 - Fix various processing errors in the Suricata module. {pull}23236[23236]
+- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 
 *Heartbeat*
 
@@ -833,6 +834,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
 - Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
 - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
+- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml
@@ -52,4 +52,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/aws/s3access/config/file.yml b/x-pack/filebeat/module/aws/s3access/config/file.yml
@@ -11,4 +11,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml
@@ -1,6 +1,12 @@
 description: "Pipeline for s3 server access logs"
 
 processors:
+  - set:
+      field: event.category
+      value: web
+  - append:
+      field: event.type
+      value: access
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
@@ -24,6 +30,12 @@ processors:
         S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+"
         S3VERSION: "[a-zA-Z0-9.]+"
 
+  - grok:
+      field: aws.s3access.request_uri
+      ignore_failure: true
+      patterns:
+        - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}'
+
   - append:
       if: "ctx?.aws?.s3access?.bucket_owner != null"
       field: related.user
@@ -99,10 +111,25 @@ processors:
       field: event.outcome
       value: success
 
-  - set:
-      field: event.duration
-      value: "{{aws.s3access.total_time}}"
-      ignore_empty_value: true
+  - convert:
+      field: aws.s3access.bytes_sent
+      target_field: http.response.body.bytes
+      type: long
+      ignore_failure: true
+
+  - convert:
+      field: aws.s3access.total_time
+      target_field: event.duration
+      type: long
+      ignore_failure: true
+
+  - script:
+      lang: painless
+      if: ctx.event?.duration != null
+      params:
+        MS_TO_NS: 1000000
+      source: >-
+        ctx.event.duration *= params.MS_TO_NS;
 
   - set:
       field: http.request.referrer
@@ -137,13 +164,18 @@ processors:
       field: event.kind
       value: event
 
+  #
+  # Save original message into event.original
+  #
+  - rename:
+      field: "message"
+      target_field: "event.original"
+
   #
   # Remove temporary fields
   #
   - remove:
-      field:
-        - message
-        - _temp_
+      field: _temp_
       ignore_missing: true
 
 on_failure:

diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json
@@ -23,12 +23,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.LOCATION",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "17",
+        "event.duration": 17000000,
         "event.id": "44EE8651683CB4DA",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -38,7 +43,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 142,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
         "related.ip": [
@@ -54,6 +62,7 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?location&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -86,12 +95,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.LOCATION",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "3",
+        "event.duration": 3000000,
         "event.id": "E26222010BCC32B6",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -101,7 +115,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 142,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 715,
         "related.ip": [
@@ -117,6 +134,7 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?location&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -150,12 +168,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.BUCKET",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "2",
+        "event.duration": 2000000,
         "event.id": "4DD6D17D1C5C401C",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -165,7 +188,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 265,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 1429,
         "related.ip": [
@@ -181,6 +207,7 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -213,12 +240,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.LOCATION",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "4",
+        "event.duration": 4000000,
         "event.id": "706992E2F3CC3C3D",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -228,7 +260,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 142,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 2161,
         "related.ip": [
@@ -244,6 +279,7 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?location&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -274,11 +310,16 @@
         "client.user.id": "arn:aws:iam::123456:user/[email protected]",
         "cloud.provider": "aws",
         "event.action": "BATCH.DELETE.OBJECT",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
         "event.id": "8CD7A4A71E2E5C9E",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/[email protected] 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Teruel",
         "geo.continent_name": "Europe",
@@ -327,11 +368,16 @@
         "client.user.id": "arn:aws:iam::123456:user/[email protected]",
         "cloud.provider": "aws",
         "event.action": "BATCH.DELETE.OBJECT",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
         "event.id": "6CE38F1312D32BDD",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/[email protected] 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Denver",
         "geo.continent_name": "North America",