diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 062fae87a46..a45c3b13b84 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -263,6 +263,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908] - Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983] - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048] +- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index e9bd3feb16f..8ec5edb4c9d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1488,6 +1488,120 @@ type: flattened -- +[float] +=== digest + +Fields from Cloudtrail Digest Logs + + +*`aws.cloudtrail.digest.log_files`*:: ++ +-- +A list of Logfiles contained in the digest. + +type: nested + +-- + +*`aws.cloudtrail.digest.start_time`*:: ++ +-- +The starting UTC time range that the digest file covers, taking as a reference the time in which log files have been delivered by CloudTrail. + +type: date + +-- + +*`aws.cloudtrail.digest.end_time`*:: ++ +-- +The ending UTC time range that the digest file covers, taking as a reference the time in which log files have been delivered by CloudTrail. + +type: date + +-- + +*`aws.cloudtrail.digest.s3_bucket`*:: ++ +-- +The name of the Amazon S3 bucket to which the current digest file has been delivered. + +type: keyword + +-- + +*`aws.cloudtrail.digest.s3_object`*:: ++ +-- +The Amazon S3 object key (that is, the Amazon S3 bucket location) of the current digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.newest_event_time`*:: ++ +-- +The UTC time of the most recent event among all of the events in the log files in the digest. + +type: date + +-- + +*`aws.cloudtrail.digest.oldest_event_time`*:: ++ +-- +The UTC time of the oldest event among all of the events in the log files in the digest. + +type: date + +-- + +*`aws.cloudtrail.digest.previous_s3_bucket`*:: ++ +-- +The Amazon S3 bucket to which the previous digest file was delivered. + +type: keyword + +-- + +*`aws.cloudtrail.digest.previous_hash_algorithm`*:: ++ +-- +The name of the hash algorithm that was used to hash the previous digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.public_key_fingerprint`*:: ++ +-- +The hexadecimal encoded fingerprint of the public key that matches the private key used to sign this digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.signature_algorithm`*:: ++ +-- +The algorithm used to sign the digest file. + +type: keyword + +-- + +*`aws.cloudtrail.insight_details`*:: ++ +-- +Shows information about the underlying triggers of an Insights event, such as event source, user agent, statistics, API name, and whether the event is the start or end of the Insights event. + +type: flattened + +-- + [float] === cloudwatch diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 1024656f082..8891e38fcc4 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -67,6 +67,31 @@ type will not be checked. If a file has "application/json" content-type, `expand_event_list_from_field` becomes required to read the json file. +[float] +==== `file_selectors` + +If the SQS queue will have events that correspond to files that +{beatname_uc} shouldn't process `file_selectors` can be used to limit +the files that are downloaded. This is a list of selectors which are +made up of `regex` and `expand_event_list_from_field` options. The +`regex` should match the S3 object key in the SQS message, and the +optional `expand_event_list_from_field` is the same as the global +setting. If `file_selectors` is given, then any global +`expand_event_list_from_field` value is ignored in favor of the ones +specified in the `file_selectors`. Regex syntax is the same as the Go +language. Files that don't match one of the regexes won't be +processed. + +["source", "yml"] +---- +file_selectors: + - regex: '^AWSLogs/\d+/CloudTrail/' + expand_event_list_from_field: 'Records' + - regex: '^AWSLogs/\d+/CloudTrail-Digest' +``` +---- + + [float] ==== `api_timeout` diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 4d3b8645a8d..ab22da1f2b6 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -102,6 +102,18 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/s3/config.go index 72960ad9ade..f9780d82277 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/s3/config.go @@ -6,6 +6,7 @@ package s3 import ( "fmt" + "regexp" "time" "github.com/elastic/beats/v7/filebeat/harvester" @@ -19,6 +20,14 @@ type config struct { AwsConfig awscommon.ConfigAWS `config:",inline"` ExpandEventListFromField string `config:"expand_event_list_from_field"` APITimeout time.Duration `config:"api_timeout"` + FileSelectors []FileSelectorCfg `config:"file_selectors"` +} + +// FileSelectorCfg defines type and configuration of FileSelectors +type FileSelectorCfg struct { + RegexString string `config:"regex"` + Regex *regexp.Regexp `config:",ignore"` + ExpandEventListFromField string `config:"expand_event_list_from_field"` } func defaultConfig() config { @@ -40,5 +49,12 @@ func (c *config) Validate() error { return fmt.Errorf("api timeout %v needs to be larger than"+ " 0s and smaller than half of the visibility timeout", c.APITimeout) } + for i := range c.FileSelectors { + r, err := regexp.Compile(c.FileSelectors[i].RegexString) + if err != nil { + return err + } + c.FileSelectors[i].Regex = r + } return nil } diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 65984dace45..1085c9dccbd 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -74,10 +74,11 @@ type s3Input struct { } type s3Info struct { - name string - key string - region string - arn string + name string + key string + region string + arn string + expandEventListFromField string } type bucket struct { @@ -252,7 +253,7 @@ func (p *s3Input) processor(queueURL string, messages []sqs.Message, visibilityT func (p *s3Input) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, wg *sync.WaitGroup, errC chan error) { defer wg.Done() - s3Infos, err := handleSQSMessage(message) + s3Infos, err := p.handleSQSMessage(message) if err != nil { p.logger.Error(errors.Wrap(err, "handleSQSMessage failed")) return @@ -352,7 +353,7 @@ func getRegionFromQueueURL(queueURL string) (string, error) { } // handle message -func handleSQSMessage(m sqs.Message) ([]s3Info, error) { +func (p *s3Input) handleSQSMessage(m sqs.Message) ([]s3Info, error) { msg := sqsMessage{} err := json.Unmarshal([]byte(*m.Body), &msg) if err != nil { @@ -361,21 +362,40 @@ func handleSQSMessage(m sqs.Message) ([]s3Info, error) { var s3Infos []s3Info for _, record := range msg.Records { - if record.EventSource == "aws:s3" && strings.HasPrefix(record.EventName, "ObjectCreated:") { - // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" - filename, err := url.QueryUnescape(record.S3.object.Key) - if err != nil { - return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) - } + if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { + return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + } + // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" + filename, err := url.QueryUnescape(record.S3.object.Key) + if err != nil { + return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) + } + if len(p.config.FileSelectors) == 0 { s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: p.config.ExpandEventListFromField, }) - } else { - return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + continue + } + + for _, fs := range p.config.FileSelectors { + if fs.Regex == nil { + continue + } + if fs.Regex.MatchString(filename) { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: fs.ExpandEventListFromField, + }) + break + } } } return s3Infos, nil @@ -456,7 +476,7 @@ func (p *s3Input) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3C } // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config - if resp.ContentType != nil && *resp.ContentType == "application/json" || p.config.ExpandEventListFromField != "" { + if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { decoder := json.NewDecoder(reader) err := p.decodeJSON(decoder, objectHash, info, s3Ctx) if err != nil { @@ -537,10 +557,10 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3 func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { switch f := jsonFields.(type) { case map[string][]interface{}: - if p.config.ExpandEventListFromField != "" { - textValues, ok := f[p.config.ExpandEventListFromField] + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] if !ok { - err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) p.logger.Error(err) return offset, err } @@ -555,10 +575,10 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash return offset, nil } case map[string]interface{}: - if p.config.ExpandEventListFromField != "" { - textValues, ok := f[p.config.ExpandEventListFromField] + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] if !ok { - err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) p.logger.Error(err) return offset, err } diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/input_test.go index c9ff86780de..d1fab05cb3c 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/input_test.go @@ -120,9 +120,10 @@ func TestHandleMessage(t *testing.T) { }, } + p := &s3Input{context: &channelContext{}} for _, c := range casesPositive { t.Run(c.title, func(t *testing.T) { - s3Info, err := handleSQSMessage(c.message) + s3Info, err := p.handleSQSMessage(c.message) assert.NoError(t, err) assert.Equal(t, len(c.expectedS3Infos), len(s3Info)) if len(s3Info) > 0 { @@ -155,7 +156,7 @@ func TestHandleMessage(t *testing.T) { for _, c := range casesNegative { t.Run(c.title, func(t *testing.T) { - s3Info, err := handleSQSMessage(c.message) + s3Info, err := p.handleSQSMessage(c.message) assert.Error(t, err) assert.Nil(t, s3Info) }) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index 7c9c9f0e7b0..b7e0c25b674 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -5,6 +5,18 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml index d4bd04b548b..155908315e9 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml @@ -223,3 +223,70 @@ description: >- Identifies the service event, including what triggered the event and the result. + - name: digest + type: group + description: >- + Fields from Cloudtrail Digest Logs + fields: + - name: log_files + type: nested + description: >- + A list of Logfiles contained in the digest. + - name: start_time + type: date + description: >- + The starting UTC time range that the digest file covers, + taking as a reference the time in which log files have + been delivered by CloudTrail. + - name: end_time + type: date + description: >- + The ending UTC time range that the digest file covers, + taking as a reference the time in which log files have + been delivered by CloudTrail. + - name: s3_bucket + type: keyword + description: >- + The name of the Amazon S3 bucket to which the current + digest file has been delivered. + - name: s3_object + type: keyword + description: >- + The Amazon S3 object key (that is, the Amazon S3 bucket + location) of the current digest file. + - name: newest_event_time + type: date + description: >- + The UTC time of the most recent event among all of the + events in the log files in the digest. + - name: oldest_event_time + type: date + description: >- + The UTC time of the oldest event among all of the events + in the log files in the digest. + - name: previous_s3_bucket + type: keyword + description: >- + The Amazon S3 bucket to which the previous digest file was + delivered. + - name: previous_hash_algorithm + type: keyword + description: >- + The name of the hash algorithm that was used to hash the + previous digest file. + - name: public_key_fingerprint + type: keyword + description: >- + The hexadecimal encoded fingerprint of the public key that + matches the private key used to sign this digest file. + - name: signature_algorithm + type: keyword + description: >- + The algorithm used to sign the digest file. + - name: insight_details + type: flattened + description: >- + Shows information about the underlying triggers of an Insights + event, such as event source, user agent, statistics, API name, + and whether the event is the start or end of the Insights + event. diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 2a6f38d1fad..2094f77c712 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -1,6 +1,19 @@ type: s3 queue_url: {{ .queue_url }} -expand_event_list_from_field: Records +file_selectors: +{{ if .process_cloudtrail_logs }} + - regex: '^AWSLogs/\d+/CloudTrail/' + expand_event_list_from_field: 'Records' +{{ end }} + +{{ if .process_digest_logs }} + - regex: '^AWSLogs/\d+/CloudTrail-Digest/' +{{ end }} + +{{ if .process_insight_logs }} + - regex: '^AWSLogs/\d+/CloudTrail-Insight/' + expand_event_list_from_field: 'Records' +{{ end }} {{ if .credential_profile_name }} credential_profile_name: {{ .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index e22c986be6f..8421e12d7f0 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -4,6 +4,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + - set: + field: event.created + value: '{{@timestamp}}' - rename: field: "message" target_field: "event.original" @@ -58,7 +61,7 @@ processors: field: "json.userIdentity.sessionContext.sessionIssuer.type" target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type" ignore_failure: true -# userIdentity.sessionIssuer.userName is only set with assumed roles. + # userIdentity.sessionIssuer.userName is only set with assumed roles. - rename: field: "json.userIdentity.sessionContext.sessionIssuer.userName" target_field: "user.name" @@ -88,6 +91,11 @@ processors: value: "{{json.eventName}}" ignore_failure: true ignore_empty_value: true + - set: + field: "event.action" + value: "{{json.eventCategory}}" + ignore_failure: true + ignore_empty_value: true - rename: field: "json.awsRegion" target_field: "cloud.region" @@ -119,9 +127,9 @@ processors: target_field: source.as.number ignore_missing: true - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true - user_agent: field: "json.userAgent" target_field: "user_agent" @@ -141,7 +149,7 @@ processors: - rename: field: json.requestParameters target_field: "aws.cloudtrail.flattened.request_parameters" - if: ctx.json.requestParameters != null + if: ctx?.json?.requestParameters != null - script: lang: painless source: | @@ -152,7 +160,7 @@ processors: - rename: field: json.responseElements target_field: "aws.cloudtrail.flattened.response_elements" - if: ctx.json.responseElements != null + if: ctx?.json?.responseElements != null - script: lang: painless source: | @@ -214,7 +222,7 @@ processors: - rename: field: json.serviceEventDetails target_field: "aws.cloudtrail.flattened.service_event_details" - if: ctx.json.serviceEventDetails != null + if: ctx?.json?.serviceEventDetails != null - script: lang: painless source: | @@ -615,6 +623,81 @@ processors: def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); + - rename: + field: "json.awsAccountId" + target_field: "cloud.account.id" + ignore_failure: true + - rename: + field: "json.previousDigestS3Object" + target_field: "file.path" + ignore_failure: true + - rename: + field: "json.previousDigestSignature" + target_field: "file.hash.sha256" + if: >- + ctx?.json?.previousDigestHashAlgorithm != null && ctx.json.previousDigestHashAlgorithm == 'SHA-256' + - append: + field: "related.hash" + value: "{{json.previousDigestSignature}}" + if: "ctx?.json?.previousDigestSignature != null" + - rename: + field: "json.logFiles" + target_field: "aws.cloudtrail.digest.log_files" + ignore_failure: true + - date: + field: "json.digestStartTime" + target_field: "aws.cloudtrail.digest.start_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "@timestamp" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "aws.cloudtrail.digest.end_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.digestS3Bucket" + target_field: "aws.cloudtrail.digest.s3_bucket" + ignore_failure: true + - date: + field: "json.newestEventTime" + target_field: "aws.cloudtrail.digest.newest_event_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.oldestEventTime" + target_field: "aws.cloudtrail.digest.oldest_event_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.previousDigestS3Bucket" + target_field: "aws.cloudtrail.digest.previous_s3_bucket" + ignore_failure: true + - rename: + field: "json.previousDigestHashAlgorithm" + target_field: "aws.cloudtrail.digest.previous_hash_algorithm" + ignore_failure: true + - rename: + field: "json.publicKeyFingerprint" + target_field: "aws.cloudtrail.digest.public_key_fingerprint" + ignore_failure: true + - rename: + field: "json.digestSignatureAlgorithm" + target_field: "aws.cloudtrail.digest.signature_algorithm" + ignore_failure: true + - rename: + field: "json.insightDetails" + target_field: "aws.cloudtrail.insight_details" + ignore_failure: true - remove: field: - "json" diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 2878c79936d..732967ff0b0 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -15,6 +15,12 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: process_cloudtrail_logs + default: true + - name: process_digest_logs + default: true + - name: process_insight_logs + default: true ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log new file mode 100644 index 00000000000..f3393babceb --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log @@ -0,0 +1 @@ +{"awsAccountId":"123456789123","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"alice-bucket","digestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"47aaa19f7eec22e9bd0b5e58cfade8cb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"alice-bucket","previousDigestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987","logFiles":[{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz","hashValue":"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:21:28Z","oldestEventTime":"2020-09-11T19:21:28Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz","hashValue":"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:51:21Z","oldestEventTime":"2020-09-11T18:51:21Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz","hashValue":"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:46:45Z","oldestEventTime":"2020-09-11T18:46:45Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz","hashValue":"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:01:06Z","oldestEventTime":"2020-09-11T19:01:06Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz","hashValue":"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:16:12Z","oldestEventTime":"2020-09-11T19:16:12Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz","hashValue":"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:56:05Z","oldestEventTime":"2020-09-11T18:56:05Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz","hashValue":"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:06:31Z","oldestEventTime":"2020-09-11T19:06:31Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz","hashValue":"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:41:58Z","oldestEventTime":"2020-09-11T18:41:58Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz","hashValue":"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:37:10Z","oldestEventTime":"2020-09-11T18:37:10Z"}]} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json new file mode 100644 index 00000000000..e4cf4e32a06 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json @@ -0,0 +1,126 @@ +[ + { + "@timestamp": "2020-09-11T19:36:49.000Z", + "aws.cloudtrail.digest.end_time": "2020-09-11T19:36:49.000Z", + "aws.cloudtrail.digest.log_files": [ + { + "hashAlgorithm": "SHA-256", + "hashValue": "420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c", + "newestEventTime": "2020-09-11T19:26:24Z", + "oldestEventTime": "2020-09-11T19:26:24Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196", + "newestEventTime": "2020-09-11T19:11:18Z", + "oldestEventTime": "2020-09-11T19:11:18Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70", + "newestEventTime": "2020-09-11T18:32:04Z", + "oldestEventTime": "2020-09-11T18:32:04Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689", + "newestEventTime": "2020-09-11T19:21:28Z", + "oldestEventTime": "2020-09-11T19:21:28Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097", + "newestEventTime": "2020-09-11T18:51:21Z", + "oldestEventTime": "2020-09-11T18:51:21Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b", + "newestEventTime": "2020-09-11T18:46:45Z", + "oldestEventTime": "2020-09-11T18:46:45Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5", + "newestEventTime": "2020-09-11T19:01:06Z", + "oldestEventTime": "2020-09-11T19:01:06Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7", + "newestEventTime": "2020-09-11T19:16:12Z", + "oldestEventTime": "2020-09-11T19:16:12Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba", + "newestEventTime": "2020-09-11T18:56:05Z", + "oldestEventTime": "2020-09-11T18:56:05Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7", + "newestEventTime": "2020-09-11T19:06:31Z", + "oldestEventTime": "2020-09-11T19:06:31Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6", + "newestEventTime": "2020-09-11T18:41:58Z", + "oldestEventTime": "2020-09-11T18:41:58Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771", + "newestEventTime": "2020-09-11T18:37:10Z", + "oldestEventTime": "2020-09-11T18:37:10Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz" + } + ], + "aws.cloudtrail.digest.newest_event_time": "2020-09-11T19:26:24.000Z", + "aws.cloudtrail.digest.oldest_event_time": "2020-09-11T18:32:04.000Z", + "aws.cloudtrail.digest.previous_hash_algorithm": "SHA-256", + "aws.cloudtrail.digest.previous_s3_bucket": "alice-bucket", + "aws.cloudtrail.digest.s3_bucket": "alice-bucket", + "aws.cloudtrail.digest.signature_algorithm": "SHA256withRSA", + "aws.cloudtrail.digest.start_time": "2020-09-11T18:36:49.000Z", + "cloud.account.id": "123456789123", + "event.dataset": "aws.cloudtrail", + "event.kind": "event", + "event.module": "aws", + "event.original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", + "event.type": "info", + "file.hash.sha256": "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987", + "file.path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", + "fileset.name": "cloudtrail", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log new file mode 100644 index 00000000000..c5c536fe7a6 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.07","eventTime":"2020-09-09T23:00:00Z","awsRegion":"us-east-1","eventID":"41ed77ca-d659-b45a-8e9a-74e504300007","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"e672c2b1-e71a-4779-f96c-02da7bb30d2e","insightDetails":{"state":"End","eventSource":"iam.amazonaws.com","eventName":"AttachUserPolicy","insightType":"ApiCallRateInsight","insffightContext":{"statistics":{"baseline":{"average":0.0},"insight":{"average":2.0},"insightDuration":1,"baselineDuration":11459},"attributions":[{"attribute":"userIdentityArn","insight":[{"value":"arn:aws:iam::123456789012:user/Alice","average":2.0}],"baseline":[]},{"attribute":"userAgent","insight":[{"value":"console.amazonaws.com","average":2.0}],"baseline":[]},{"attribute":"errorCode","insight":[{"value":"null","average":2.0}],"baseline":[]}]}},"eventCategory":"Insight"} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json new file mode 100644 index 00000000000..2bfe4bedd30 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json @@ -0,0 +1,64 @@ +[ + { + "@timestamp": "2020-09-09T23:00:00.000Z", + "aws.cloudtrail.event_type": "AwsCloudTrailInsight", + "aws.cloudtrail.event_version": "1.07", + "aws.cloudtrail.insight_details.eventName": "AttachUserPolicy", + "aws.cloudtrail.insight_details.eventSource": "iam.amazonaws.com", + "aws.cloudtrail.insight_details.insffightContext.attributions": [ + { + "attribute": "userIdentityArn", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "arn:aws:iam::123456789012:user/Alice" + } + ] + }, + { + "attribute": "userAgent", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "console.amazonaws.com" + } + ] + }, + { + "attribute": "errorCode", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "null" + } + ] + } + ], + "aws.cloudtrail.insight_details.insffightContext.statistics.baseline.average": 0.0, + "aws.cloudtrail.insight_details.insffightContext.statistics.baselineDuration": 11459, + "aws.cloudtrail.insight_details.insffightContext.statistics.insight.average": 2.0, + "aws.cloudtrail.insight_details.insffightContext.statistics.insightDuration": 1, + "aws.cloudtrail.insight_details.insightType": "ApiCallRateInsight", + "aws.cloudtrail.insight_details.state": "End", + "aws.cloudtrail.recipient_account_id": "123456789012", + "cloud.region": "us-east-1", + "event.action": "Insight", + "event.dataset": "aws.cloudtrail", + "event.id": "41ed77ca-d659-b45a-8e9a-74e504300007", + "event.kind": "event", + "event.module": "aws", + "event.original": "{\"eventVersion\":\"1.07\",\"eventTime\":\"2020-09-09T23:00:00Z\",\"awsRegion\":\"us-east-1\",\"eventID\":\"41ed77ca-d659-b45a-8e9a-74e504300007\",\"eventType\":\"AwsCloudTrailInsight\",\"recipientAccountId\":\"123456789012\",\"sharedEventID\":\"e672c2b1-e71a-4779-f96c-02da7bb30d2e\",\"insightDetails\":{\"state\":\"End\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AttachUserPolicy\",\"insightType\":\"ApiCallRateInsight\",\"insffightContext\":{\"statistics\":{\"baseline\":{\"average\":0.0},\"insight\":{\"average\":2.0},\"insightDuration\":1,\"baselineDuration\":11459},\"attributions\":[{\"attribute\":\"userIdentityArn\",\"insight\":[{\"value\":\"arn:aws:iam::123456789012:user/Alice\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"userAgent\",\"insight\":[{\"value\":\"console.amazonaws.com\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"errorCode\",\"insight\":[{\"value\":\"null\",\"average\":2.0}],\"baseline\":[]}]}},\"eventCategory\":\"Insight\"}", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "cloudtrail", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 5b8a3d61898..352932f1b1c 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXF9z4zZwf79PsZOX2DOSOrlkOh130hmdz9eocS6upUvaJwYCVxJqCGAA0Drdp+8sAFKkSEqyReVOD4nPpBa/Xex/LDyEJ9zeANvYNwBOOIk3MP5z+gbAoERm8Qbm6NgbgBQtNyJzQqsb+I83AAC/6TSXCAttYMVUKoVagtRLCwuj10Rm9AZgIVCm9sZ/YQiKrbFYjj5um+ENLI3Os/iblnXo88GTKSn7dUbxaXWJ6jJc6jx1hglZPmpbkT773BafFBcsly7xS9zAgkmLtcetYKuAtfF4bwnLjLDUoLfBr7KAz6hc8ozGCq1qbxScPOF2o0269+wAMPrMVlhFFOmDXoBbIQEMCxP6NXOjVmi5RZOIFJUTbtsKbV/ITWDDVmREeRIJA0pcExSulWNCWUjRMSEtsLnOncdLq4FeNGhNxr9BARDcijlYsxT9Vwz+naN1A2Aqhc1K8BVwg/5dJi1s0GCDXG4xHcFkAQ7XmTbMbBvf8e8M/AoFbrvSGwsrvaHfNmg2COg5cYnpaO/VNiWp7gbJoPHwsI40t6PlhbAjUcKesY4tr1i32dfUlyNpKkYBZbxmX7SCR7Q6NxzhI1sjXI0fP14XADMjFBcZk3t7zpmU+2KtoOYcrU2ecJuINnx94Q/rECGYvA8IN8x6xQGnwYqlqmpoN2CLlow2IcPAz64TcpsVngp4sqhi8UC9ODfCrSpmYJHnpk0loK7iZG6lYXjWM6OfRYoWhAq+htzQzrIjj610S9Fxg8xh6l2tW2mL1SVbvtplSlXhrhcsYblbERVO1FvfPq4VpwoaonY8M5kjCAvO0P+j+LV23imCNt6p+Z83xGonsVbPFEW021AmrfYyrPEatpe1i50+v30YQ4rPguO/g3YrNBthcRCiY1Nhq3L1e0VamzLXBT7I9MALLxEokfFO3ok1wmaFwbqautuUmLA2bzriOj+FEfp3zUGGuuzwJRz1YY/wepvspBfD2+nhrPgcskU4Ht6KzylmCC+QM0TtiTEmBpbDSjMAm/PVQZLMwqPWbkBG/MmiGZBBP2rZYTRVAZRBrT06XVoQQjk0ikmKWVEa1byqGsGW2K0nUNe942y35xKX5nb8+LHgMmrAFeNc5ypsnfe/fu+Mlnh9kFybeI4o0glSCWC+jirExQNneqPs5bSh4FeoZ/2EaTJv82h9JWa0VLHrVLFZNBThugoHMnZgbfkFFDnq3e1bGOdOw5QzXxzHWvBOMusEh3fIlHVMPrUXWGiMNgnX6b7nO73wa6+vqtz5RcpEI8YVgy43yvrIQM8P4VujtWzZJ8TJYTChvKoQKR1SN9RIK8mYYWt0aPb37VyR7ggPSJhMbQfRFigKWoqtIUZ3Z/brXDqRHK3zDqX6HQ+PNDGqYrKZVhaTmA/0LaWCfplvUD7KOH3HFnb2hMBXTC3RwlXI7AfNSjyjtM574BQlUoYXiFx/g0JlaSoIHJOJb6qkrNZhOleu45I85bqsUjyFFk7pkZV2pKYumkuDUGFvFNbiPn2D0iwsuRH6ztXNUHl5aYUOx0KgrdlryAij35yjUMtmO4dJiSksUaFhzn9f2EC6w4f6/l5LdnuWA63jL5o3FX0oAKYVRTHItUnbYbJMnN2EPIpz/DApO5HMWs3Frhj1zzd2nIlbJmWDkudgRnwekPWaKbb0ficYYp9GCO+0lshUhxptVkhlckXawsK+F4AKwvBWVzBjaaKVbG+6nr0VO6zCgs5IT2hHCLBfekhL7x50YQyJc3sEeU1jeAxSWO+8Stqxl4YpCLUT7Us7p5drV5Y9yvHjx2aieFIy3weMcUzYd6VbIUFK31u6Fr32lI+Ixjun0lJMsZOh8eDPom5u2MYOo98demQ3FOmG9FX/7w4N5CITZOydAj7HYB4xM0h5XfBdbCdjb/sGOYpn71+FPWTMka/gkZJ4tnE5H1um/bTcAITiMk+pNNkQamfEcokmhIV2Jxt6aUGHcvktJrF2xQymUaC97vl/fpq8r4TO+bZ6huY05Er8naPcFvpcfd4uzXig6XeGyk+qzEIqG0OIDbWD05CKxQIN/SOcz9Y/Uf9su5I9ZzxBlWZa9C2SPfX64+EWioXIlMPJWkygYkvQl9Ke7WYApO87DUz5rnK1UC0L7qK4nv7YzivXymqJidRL0Z6svCb6xNNcmyEXC8EJ5G1Y6J7Wibv50shzvDI4jLqJvD0A7EqEO2/D76lQoNrrOA+H+KjystZzIbEjSaxzMg+pUsc7J7fCmxlLjRsqebyWBWRwCFnBhNeZxOmD8Ps57Pn0eN/YgcMCXrAkt0cOoi4mWu/XYcG4o4p9d1hEnmrT0QWDeGYOaW4oxHSyWrC4kMw5VA0eX2+2d9MdUV8e+3SD5K7n/4fceQZN6OzZfB4UHZhB9b2DJ6U3irwYS5+Z4o1quFfz7uK9yeUxEz+vCwCndQJe0Fp7DY/dXdoeGm2n97v6hd5X9wte1AF7WarZH8c9Jp5wNPmsTXxtmKudw33Nia8/CcuLJr4O9dNbUuMjvqEOoiDeFBzyt19fYne3b8P8oFAV4KcKTmQJS1OD9vXlU0N8u6NPdBCp74po713QtEhTzk+X5vIysrx/9yK1a9TRZ8utep4mNUthziSF0Y4jmrN6oK0AqkNrNQBeSM9v4Z5++S7+sqN0csws0SV+90bNVtGZECsHzWGhoCa7cdrOMFamrMKSl95v4pyJy2tPpNxoaCDVWAp5d/cvM9pprveL3jNBFVTb9/Rq5VxG8dDx7PrIKaDRHK0Vapk4scaRRd6KdCE1e6G/9XqnHZNh3kgosMi1otJRKI570gsZVki2hN3JOFdOSBC1+p+BwSVtCUXJOeNPqDqa9fHhN8RmhQ16EgGCE1LWfmEdM87GRImyg87mckjbvjKHZUJX3buyoVPjsr6XXjxSdDYFd6ILfK3bQ5rUjSOol3EVDam6UwrWQkoRmR1EbgN8nfk6qsIQl9ruD3eVnlPahPyYXbEnvCwfxVjj7H4K5ZIkaK7Xmc+O9/gC3aKlK2ZhjqgArWNzKeyqi7XC/MR+PXqmh5s87CcZhRLtND10lI954AJhps3rz7navbA2rujSn4uOXPaoLDGtYy635w28tEIOhIEIN83zqvq0aaDhCxQPyX6dNuG06a9O6H+1cmytTLjIVn0H6un0HgLd0GoRiozgX/yvy03oSG0I02UiNS1fRusX4+IrbVElHI27aMYV1gFaRyz85DPEM50wulZRgtfCt2gE61m4gSaofD1Hc2FehOJ67cOrtAmT2LczodpqiSaemuuF9+B+nUpEnW/bUvjCjgNXvhkUOe4OR/RTmrQ1Es/kg2B74iF/70BgGMdzjl9al/bXP5QrI8Zf/zMcr7+o4YxWG07Sv2CFLO2qutZUYWOamFxSOiV0y9j42Wl7oLrbZJ875bIcrfQQ6tOVYkEJL70TH3fMpYQkCT8jz5uXNM4EHg/iCuLh8kAt1O2GFK8W2myYSQewEJ8xHRaRYVC7WTEaja5HMHHAmSoutIHFZzRMBvF02KHBVBjkLslNz97k0+N99NBe4nEdP4LDi+5+KYIDc6Ajg8z2fUUxDKcGysWMarkdEd+CCVlFVka3H8N0yNduxEx/9L1PNMXNr5d0ZeY5f0KXtI1pnOs2mNJKcCbDZaLdbIhfa2/6PcDoyOX8s8u1jgL94Cr2775EYyRlWDKhunokBtfaYdKRrTd+fYpryDJmQjA9uTkIzU7EP7ap5Xr+xIDB0KtnrurXvopz+sPdkz4D2Bis82eDtamK8oS/fagC2W6moB1qOZzWr3h3w3C+N5bCKlwTgxS5ZFQZMAvT38cPo/LNATzeTWejX2azh2SNbqXTUTF85ac+B/Dn3bvpZHZ36BVt4N14dvvL6P3d/d3sbvT7u/+6u521s/6EPYfv755w+131yHAXpCl2oKKiJ/Ugvxt+V3jpnahSjeHU0VFNzvzF1/IQ77Cm5Ub0y8tjIDz89DipcUSyLx1L44ykCo3KviRUez22MlS+RiN4wFGtR3fzei2nsD3eE2kvlUozvPNx+FanWN1npWOA1pznxnQ2S7YObWK7Bm5fLbFY/ZRNOL+Or+AHgJ+L40Uv0l3n+BkNpcNVNr6g0R1uxM8IJFZ8aZfsOX0rIlqGXL8OFWZeAzvbJv6bvpN2EUnWGmT7oVYoWEixXLnKxJpPa763kKGxGWWNzx0a6nKjEmZ0rtJ/DD5zFQW2GQXrSu6+1bk5HEMMLtCYviN0rQbyuvkY14ll2vFC1v/dDbY8Z4D9RGifLJrhmFY6WEPGyareK9viLsDkfdFvLCPPqcEmkpikx0LOSvec2RADn4ds/WUo0uFbf/Ov1Eb87FClu4QLJu87OnRiqZjLTddc3bmdpIJ8IacBTMXyD4+Wfvhp0LzPWc0Ya07i1XllaF8mNheNC//ntiGRE3NTTRWEhXu2RQNX0+n9ddEz3Y1F4VI7Uf4ZCVL/aRtr9KCjFVEbijvvSk973IjH2eXfbqhP4YW/PDPO3eoXb6thuqj+TrBiO4D/ztFspyH1pvf+pn8XufhVZnBIuoEppXjXr99ab1Vh0Z4bA8WMcaGWsZNJPx6ZD3bSXsaaZoYp609HgqJNi6vdV7P76XXpzSqaFhub+yeBlanthdSb0xsYl5rL+ePhFgjJi1oXF5ExIflASO710hZL+L/JtNU5KUO8G+wZj5dIwn3hQvzCwtvyC2GcewsMeG6dXnd9o0OVerhY0p54+1sA5YWS4nSz2IKuVr1Ds7hEh3nXRlDoNto87dby2MKQv59hMWyxEDyeh2uTHu7b9gtz795E2w3CiG8A49vbu4eZ/+sbd921tNTLQ7Xeq5FKvVySo42VXhRusb0D+P3XAXz8/f14NvaR+NfJA/3cte3WMXXRXS+W8KL9vinZV2jFoEjdStrC+s6jd4pbnXfMFT25xBrO0rQ9nrymlZcxyg6GEp9RwpU2YikUk9dF67N5JB/Z6UaYWvePIEypVlQhsldgFu7iIM7njF9QY/w1HrLD8o/29eo9bD5X2L/b3eEPC1ySBcezZCHZsmfPMhduzexTrOXKwKGl1BvyOLPbB/DL3sDbn6f/+3Hww7/R/4bj218HP/z8YfJx8NPPj9NZO+TLDWgGqd3A5OH5pwH99199iXf3YTx68/8BAAD//624A44=" + return "eJzcXN+T4raTf9+/oisvma0Crr6b1NXVXOWq2NnJhctkMzewyd2TI+wGdCMkR5JhyV9/1ZL8AywDM5jd1JeHXQbb0qdb/VstD+EZd7fAtuYNgOVW4C2Mf5++AdAokBm8hTla9gYgQ5Nqnluu5C38xxsAgF9UVgiEhdKwYjITXC5BqKWBhVZrGmb0BmDBUWTm1j0wBMnWWE5HH7vL8RaWWhV5+CUyD31+dMNUI7t5RuFqc4rmNKlQRWY146K6FJuRPofUlp8MF6wQNnFT3MKCCYN7l6Ngm4CVdnjvCMuMsOxBj8FvkoAblDbZoDZcyb07SkqecbdVOju4dgQYfWYrbCIK44NagF0hAfQTE/o1s6MotMKgTniG0nK7i0I7ZHIb2DCKjEaehIEBBa4JSqqkZVwayNAyLgywuSqsw0uzgVq0xpqMf4ESINgVs7BmGbpHNP5ZoLEDYDKD7YqnK0g1unuZMLBFja3hCoPZCCYLsLjOlWZ613rG3TNwM5S4zUptDazUln5tjdkaQM2JSsxGB7fGhKS5GsSD1sXjMtJejsgNfkUChx1hHUve0G59KKkvR9IWjBLKeM3+UhKe0KhCpwgf2RrhZvz08W0JMNdcpjxn4mDNUybEIVsbqNMUjUmecZfwGL6+8Pt5aCCYfPAIt8w4wQGrwPClbEpoN2CDhpQ2IcXAz7YTckwLzwU8WTSxOKCOnVtuVw01MJgWOiYSsC/ipG6VYjjSc602PEMDXHpbQ2ao1uxAY3TcinWpRmYxc6bWrpTB5pSRR7tUqcnc9YIlrLArGiWl0aN3n5aKcxkNQTo2TBQI3IDV9H9gv1LWGUVQ2hk1931LpHYOFrVMgUX1gjJhlOPhHq1+eVmc7fT55ccxZLjhKf47KLtCveUGB947tgW2yVe3ViS1GbNd4D1Pj9zwEobSMM7IW75G2K7Qa1dbdtsc48YUbUO8T0+phO5efZSgLj18CUV96CO8Xic7xwvu7Xx3Vn6O6SKcdm/l5xw1hBfwGYL0BB8THMtxoRmAKdLV0SGZgSel7ICU+JNBPSCFflKiQ2maDKicWtw7XZsRXFrUkgnyWYEbzbiq6cGW2C0nsC97p8mOxxLXpnb89LGkMkjADUtTVUi/dM7+urXTSuDbo8PF2HNCkM7gigfzdUQhTO4pU1tpricNJb1cbtQzZsk8ZtH6CsxoqnLVKWMzqMnDdSUOpOzAYvEFlDHq/d07GBdWwTRlLjkOueC9YMbyFN4jk8Yy8RxPsFBrpZNUZYeW7/zEL55fNalzk1SBRvArGm2hpXGega4fw7dGY9iyT4iT42B8etUYpDJI3VDDWEnONFujRX24bpeytB54QMxkcjcIukBe0JBv9T66O7JfF8Ly5GSedyzU77h4oojRZJPJlTSYhHigby6V41fxBsWjLKVnTKlnzwjpisklGrjxkf2gnYnnFNY5C5yhQIrw/CBv/4ZMZVnGCRwTiSuqZGyvwnQpX8fV8BTrskby5Es4lUWWypKY2qAurYFKfSO3Ftbpb8jNUpNbru9S2fSZl+OWr3AsOJo9ffURYbCbc+Ry2S7nMCEwgyVK1My657nxQ3fYUFffi0S3FxnQffxl8aYhDyXArCEoGlOlszhMlvOLi5AncY4fJ1UlkhmjUl4no+761oxzfseEaI3kKJgRnUd4vWaSLZ3d8YrYpxLCe6UEMtkhRtsVUprc4DY3cGgFoIHQ39XlzFiWKCniRdeLl6LGyg2onOSEVoQAu6mHNHV9oQujD5zjHuQ1heExCG6c8arGDrU0zIDLmrUvrZxer1xZ1SjHTx/bgeJZwXwfMMYhYK9Tt5KDFL5Hqha91pRPsMYZp0pTdLmSvvDg9qJub9nWDIPdHTpkt+TphvSo+7tDAlOec1L2TgZfojBPmGukuM7bLlbz2Om+xhT5xtlXbo4pc6DLW6Qk7G1cz8ZWYT9NNwAuU1FklJpsCbXVfLlE7d1C3Mj6WpqXoUL8HYNYs2Ias8DQXtf8Pz9NPjRc53zX3EOzCgrJ/yxQ7Ep5bl6PczNsaLqVofSTMjMfygYXYnzuYBVkfLFATX/4/dn9T5A/ExeyTZ4mKLNc8b5ZciBevz3eQTkRqbLfWQsBVCgJulTakd12gPS8VcCkqyo3E9Uq4S6T6+l3cVpTJY0SmAi15PFg5TXeJ+zmmhxTvuApgbzzEz3QPGE1X+p5TmcGx1G3kccdQJ0i3Dsd/kCJAuVep2k4RkeTlrWac4EdQeI+JXMfKnXcc3YpvB2x7FFDKY+TMo8MjiEriXAyk1h1FH4/mz2fnh5aK3CcwQuWFObERtTVWOvsOixYailjrzeLyFJtO6pgEPbMISs0uZhOUksSF4JZi7JF4+vV9n5aD+rSYxduEN/V/P8wtY5A7St7pph7QQemUX5r4VmqrSQrxrINk2krG+5Vvbtob1N5SsUvqwLAeZWAF5TWXkNjd5W2h0Lb+fWufqH3Vf2CF1XAXhZq9kdxj4EnnBV8lmRmfIkmntxf4PmdK7mrGsrgg5sFHtTypV5fqGWy4KKVGtcwJZp4x8FZ2V6VJz+opZunbJ2q82TPoiOSYpm2ieXr7vSvY3v+XF1wM9Dyf5rd+a15TVLv1aCGCEQApIqcd1wRLHumYZivTrgIOfX23I1axaBCLcFzY8U2cRGbI0rSJr5xkrgX3nfzCmV2VU6hzP4p+GS+S+ZF+hzdDrzKFl6ZJoCflkJ2T6JrVCi0bpcAw2wNlq6YOaD3KIU+qrgihTVVfirXR3YTcsVBlPDoWEL54K1qmgsMaRLfTanELbn8ULy+ovBXUh9QrpXxtZWyQgpsrUishTgW0IRcO1i/WsTPNIdKZF+HXD9xB6Xd+Rq8mtJc44arwiRfQluPa2gJZU8du1KNM7SzIm3FzCphYqk0t6v1F7JGNClUk7bbJNz1LvGNseIIocVc8NQ1si64XKLONY9aur7oXOFnlmHK10wAylRlmEFj5qot1+FyBovIjw63ZjZdhYgx13zDLLoHDrpj+ZmsoNuZLTR+kfWul7fVzNuJtu5wMXy5Ol4A7orIT+Gbug70eINtITPUYkehQYjBDa0XkzDxiNr6FiL4shznjZOv5g98Zypb+jsss9xYnpqB29sjStvBiW/Aj2yP2TJMpCQHZVbK0XFgNXv3ToFsSbDeHHL065wC+Z2wvOgUyLEem0i5/ES9YB9EOXibcZi++/ocu797588UcdkAfi7jeJ6wLNNoXr+l0mJf3Q6JFsLo9caaqzigjnBTzM/n5vI6vHx4/yKxa+2tXcy3pksUimUwZ4LJFDvati7qi4gCaB5k2QPgmLR5Bw/04/vwY8d2imV6iTZxqzdqbx9fCLHRfOon8mJSH7HrLG1VFQZuyE8cbuxeiMtJTxi5tclJyaeUmHZ3BORaWZWqw42wC0GVo8bX9GZlbU7uw6b52xOdgVqlaAyXSxfhjwymHU5YtWKXc+ROWSaqVNtgqmRmwPAyCa+556uuvgDLTc3jQlougO/tCVIqv6QlcXk4S59RdjTwhIt/IzIbZNCVABAsF2LvBxcCmFA8zbhcdjac+FLuV6awKvI2167a5N2jcn8tHXsE72wUqFnn6VrHXZpQrba0l1FV1gMaKyVhzYXggdhBoNbDV7nbW2kQlAplDhOxynIKyr9kZlbsGa9LR3nUafYwhWpKYnSq1rmrmB/QBSoipVX9B41lc8HNqou0Uv34YYH5Qgs3eTwMMkohqiXdp0SnLHCJMFf69b1vcSustC1LDJeiI5M9qradKIcozGVN8FHIfmCggdvqedO82lZQ/wD5Q9Jfq7SvrP/RCf2PKMXGiCTl+apvRz2dPoAf1yeiXJIS/Iv7uVqEjtCGMF3HU9P0lbd+Ma50pQzKJEVtrxpx+XmA5uELdxoSQp+Xz+gbQvBa+AY1Zz0z148JsljPUV+ZFi5TtXbuVZiECezbmFButUQdOmnVwllwN0/Do853sRC+1GNPldsgDhR3uyP6liWxncEL6SDYbnAfv3cg0CzFS1qyolO7I+HSVh7jj/8Zjtd/yeGMZhtOsj9ghSzryrp8ES5LdCEonOIqcpT04rDdj1ovsoudClEdt3IQ9k9c8QUFvHRPuNzRq+6DJPyMadHeRr0QeGjOKwf3B4r3XF19cOlmofSW6WwAC/4Zs2HpGQZ7p61Ho9HbEUwspEyWO7VgcIOaCc+eDj3UmHGNqU0K3bM1+fT0ECy043iYxxUO07Ljp2LBkbNhI43M9P3aEn9gzY9cnlurliPgWzAumsjqHTrfMf61CzHT71w/BOrybRAvqcr4zZIk1rp9qdlgUkmeMuHLuHW/uJvr4ESsh9ERy8W2jnosHZX7ReUmePM8fFBGEoYl47KrRqJxrSwmHdF66+dzTEOeM+2d6dnFQWhXIr7YolbzuS4iBkMnnoXcfxVE2bt7vHrSpwMbg7GuX3Cv07reLIw2WiOr+4zjUKsDK/2ytz4g42pjGaz8qyMgw1QwygyYgemv48dRdecAnu6ns9FPs9ljska7UtmoPJDhToIN4Pf799PJ7P7YLUrD+/Hs7qfRh/uH+9n96Nf3/3V/N4uT/ow9u+9vnnH3TbONsHbS5DvCJqAD+c3wm9JK16zKFPpOREs5OXNbfFVj33FJKzTvl5YnP/Dw09NkjyLifWVYWnskTWiU9iU+2+uxlCGLNWqeehzNfLQ+wxPpzOzx7Hg8VarU8N754TuVYXOdpQoOWqWuo6SrWLKzaBLTdQjv1RwL2U9VhHPzuAx+APi5bDl0LK0rxxvUFA43yfgLteowI67tJjH8rzhnL6lb0aCVy/XtPVyCk8DOsol7MtaU0g8n9wpkh66WS1gIvlzZxikWF9Z8ayBHbXKKGjcdEmoLLROmVRHvn7sKfGYbAmxyctaN2H2nCn3ch7jGOd23h97LgZxsPoV5Qpp2OpF17+Jz2+7XhvbJoB6OaaajOWQ4bdF7ZlueD558KOuNlec519mEISbZKZezUj1HNkTA5yFb/zXk2fCdextIJY342aLM6oALJh86KnRVT8tV3gpZDV/yaQBTvvzNoaUv3w/azUvNiHHPSLw6rvTly8QUvNVPd2kZElMibqoogzDwwHao4WY6fXhb1kzroxK4VJZXr5Yj8Z/GSKMLHaWIvYMylx3zj/uNsJ1dvc9t/2SOfxvluLCrn5yu+hMH+/d4LTYD+O8C9W7qQ2+670/6u4zFb3KNQ5INzCjEe/v6pXVa5SftuTBQnjssxTJUMunriTODVpjraNNMM2nc7ogXtGn5uqeb2cP0bWXNGpIWCpuHO4GNk5wLobbnFzCu1Zfz2+MdEJIXlS6uwmNC8iMheVBLU07h3tO6UwUJQ3hfkCM8HCz3ndIl+7mBd9UD/ojnDhikhbFq3fVEhyj1cNg8Hni7k8HVIfNyd7Ncgq5SvUW9uEaFuS4jSLRbpZ/ruRy2uo/XarZY8DTshyudHa/b9gvz4Cx17K0iAd8Axnd3948z90a+++5cWqjlsVzv1UiFWi7J0IZMLzC3XN4B/PrzAD7++mE8GztP/PPkkb53NpNaJq+66uUUjrXftjn7CqkYlKFbNTY3rvLojOJOFR19Rc82MTplWRb3J68p5eWMooOhwA0KuFGaL7lk4m1Z+mxvyQdyuhFmxn4RhBnlitJ79gbM0lwcxbnJ0ytKjDvaT3pYvci7V+thirnE/s1ujd9PcE0SbJonC8FaBwovJGHO7ZqZ55DLVY5DCaG2ZHFmd4/gpr2Fdz9M//fj4B//Rv8Nx3c/D/7xw4+Tj4Pvf3iazuKQr9eg6bl2C5PHzfcD+vdfXYp3/+N49Ob/AwAA//+8xDjD" } diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 7a0da775c8d..0fe8465211b 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -8,6 +8,18 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows