Fix improper nesting of session_issuer in aws/cloudtrail (#18915)

pipeline & fields.yml had session issuer outside of the session context object. session issuer only appears inside the session context object. Not a breaking change because prior to this the data was being skipped. Closes #18894
elastic · Jun 5, 2020 · 632c086 · 632c086
1 parent f3ab7c7
commit 632c086
Show file tree

Hide file tree

Showing 9 changed files with 71 additions and 60 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591]
 - Fix Kubernetes Watcher goroutine leaks when input config is invalid and `input.reload` is enabled. {issue}18629[18629] {pull}18630[18630]
 - Okta module now sets the Elasticsearch `_id` field to the Okta UUID value contained in each system log to minimize the possibility of duplicating events. {pull}18953[18953]
+- Fix improper nesting of session_issuer object in aws cloudtrail fileset. {issue}18894[18894] {pull}18915[18915]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1149,22 +1149,13 @@ type: date
 
 --
 
-*`aws.cloudtrail.user_identity.invoked_by`*::
-+
---
-The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
-
-type: keyword
-
---
-
 [float]
 === session_issuer
 
 If the request was made with temporary security credentials, an element that provides information about how the credentials were obtained.
 
 
-*`aws.cloudtrail.user_identity.session_issuer.type`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.type`*::
 +
 --
 The source of the temporary security credentials, such as Root, IAMUser, or Role.
@@ -1173,7 +1164,7 @@ type: keyword
 
 --
 
-*`aws.cloudtrail.user_identity.session_issuer.principal_id`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.principal_id`*::
 +
 --
 The internal ID of the entity that was used to get credentials.
@@ -1182,7 +1173,7 @@ type: keyword
 
 --
 
-*`aws.cloudtrail.user_identity.session_issuer.arn`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.arn`*::
 +
 --
 The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials.
@@ -1191,7 +1182,7 @@ type: keyword
 
 --
 
-*`aws.cloudtrail.user_identity.session_issuer.account_id`*::
+*`aws.cloudtrail.user_identity.session_context.session_issuer.account_id`*::
 +
 --
 The account that owns the entity that was used to get credentials.
@@ -1200,6 +1191,15 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.user_identity.invoked_by`*::
++
+--
+The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
+
+type: keyword
+
+--
+
 *`aws.cloudtrail.error_code`*::
 +
 --

diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml
@@ -46,38 +46,38 @@
               type: date
               description: >-
                 The date and time when the temporary security credentials were issued.
+            - name: session_issuer
+              type: group
+              description: >-
+                If the request was made with temporary security
+                credentials, an element that provides information about
+                how the credentials were obtained.
+              fields:
+                - name: type
+                  type: keyword
+                  description: >-
+                    The source of the temporary security credentials, such
+                    as Root, IAMUser, or Role.
+                - name: principal_id
+                  type: keyword
+                  description: >-
+                    The internal ID of the entity that was used to get
+                    credentials.
+                - name: arn
+                  type: keyword
+                  description: >-
+                    The ARN of the source (account, IAM user, or role)
+                    that was used to get temporary security credentials.
+                - name: account_id
+                  type: keyword
+                  description: >-
+                    The account that owns the entity that was used to get
+                    credentials.
         - name: invoked_by
           type: keyword
           description: >-
             The name of the AWS service that made the request, such as
             Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
-        - name: session_issuer
-          type: group
-          description: >-
-            If the request was made with temporary security
-            credentials, an element that provides information about
-            how the credentials were obtained.
-          fields:
-            - name: type
-              type: keyword
-              description: >-
-                The source of the temporary security credentials, such
-                as Root, IAMUser, or Role.
-            - name: principal_id
-              type: keyword
-              description: >-
-                The internal ID of the entity that was used to get
-                credentials.
-            - name: arn
-              type: keyword
-              description: >-
-                The ARN of the source (account, IAM user, or role)
-                that was used to get temporary security credentials.
-            - name: account_id
-              type: keyword
-              description: >-
-                The account that owns the entity that was used to get
-                credentials.
     - name: error_code
       type: keyword
       description: >-

diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -52,29 +52,29 @@ processors:
       formats:
         - ISO8601
   - rename:
-      field: "json.userIdentity.invokedBy"
-      target_field: "aws.cloudtrail.user_identity.invoked_by"
-      ignore_failure: true
-  - rename:
-      field: "json.userIdentity.sessionIssuer.type"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.type"
+      field: "json.userIdentity.sessionContext.sessionIssuer.type"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type"
       ignore_failure: true
 # userIdentity.sessionIssuer.userName is only set with assumed roles.
   - rename:
-      field: "json.userIdentity.sessionIssuer.userName"
+      field: "json.userIdentity.sessionContext.sessionIssuer.userName"
       target_field: "user.name"
       ignore_failure: true
   - rename:
-      field: "json.userIdentity.sessionIssuer.principalId"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.principal_id"
+      field: "json.userIdentity.sessionContext.sessionIssuer.principalId"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id"
       ignore_failure: true
   - rename:
-      field: "json.userIdentity.sessionIssuer.arn"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.arn"
+      field: "json.userIdentity.sessionContext.sessionIssuer.arn"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.arn"
       ignore_failure: true
   - rename:
-      field: "json.userIdentity.sessionIssuer.accountId"
-      target_field: "aws.cloudtrail.user_identity.session_issuer.account_id"
+      field: "json.userIdentity.sessionContext.sessionIssuer.accountId"
+      target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.account_id"
+      ignore_failure: true
+  - rename:
+      field: "json.userIdentity.invokedBy"
+      target_field: "aws.cloudtrail.user_identity.invoked_by"
       ignore_failure: true
   - rename:
       field: "json.eventSource"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json
@@ -10,6 +10,10 @@
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2019-10-02T21:50:54.000Z",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.account_id": "111111111111",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.arn": "arn:aws:iam::111111111111:role/JohnRole1",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id": "AROAIN5ATK5U7KEXAMPLE",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.type": "Role",
         "aws.cloudtrail.user_identity.type": "AssumedRole",
         "cloud.account.id": "111111111111",
         "cloud.region": "us-east-2",
@@ -38,6 +42,7 @@
             "forwarded"
         ],
         "user.id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
+        "user.name": "JohnDoe",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",
         "user_agent.original": "aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log
@@ -1,3 +1,3 @@
 {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JohnDoe","accountId":"111122223333","userName":"JohnDoe"},"eventTime":"2014-07-16T15:49:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.110","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","requestParameters":null,"responseElements":{"ConsoleLogin":"Success"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/s3/","MFAUsed":"No"},"eventID":"3fcfb182-98f8-4744-bd45-10aEXAMPLE"}
 {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JaneDoe","accountId":"111122223333","userName":"JaneDoe"},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
-{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"}},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
+{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json
@@ -94,10 +94,10 @@
         "aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
-        "aws.cloudtrail.user_identity.session_issuer.account_id": "123456789012",
-        "aws.cloudtrail.user_identity.session_issuer.arn": "arn:aws:iam::123456789012:role/RoleToBeAssumed",
-        "aws.cloudtrail.user_identity.session_issuer.principal_id": "AROAIDPPEZS35WEXAMPLE",
-        "aws.cloudtrail.user_identity.session_issuer.type": "Role",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.account_id": "123456789012",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.arn": "arn:aws:iam::123456789012:role/RoleToBeAssumed",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id": "AROAIDPPEZS35WEXAMPLE",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.type": "Role",
         "aws.cloudtrail.user_identity.type": "AssumedRole",
         "cloud.account.id": "123456789012",
         "cloud.region": "us-east-2",
@@ -107,7 +107,7 @@
         "event.id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE",
         "event.kind": "event",
         "event.module": "aws",
-        "event.original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"}},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
+        "event.original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
         "event.outcome": "failure",
         "event.provider": "signin.amazonaws.com",
         "event.type": "info",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/delete-bucket-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/delete-bucket-json.log-expected.json
@@ -9,6 +9,10 @@
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2016-11-14T17:25:26.000Z",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.account_id": "777788889999",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.arn": "arn:aws:iam::777788889999:role/AssumeNothing",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id": "AIDAQRSTUVWXYZEXAMPLE",
+        "aws.cloudtrail.user_identity.session_context.session_issuer.type": "Role",
         "aws.cloudtrail.user_identity.type": "AssumedRole",
         "cloud.account.id": "777788889999",
         "cloud.region": "us-east-2",
@@ -31,6 +35,7 @@
             "forwarded"
         ],
         "user.id": "AIDAQRSTUVWXYZEXAMPLE:devdsk",
+        "user.name": "AssumeNothing",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",
         "user_agent.original": "[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]",