diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 165b64e3b4b..44bfb61eb6e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -839,6 +839,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `fail_on_template_error` option for httpjson input. {pull}24784[24784] - Change `okta.target` to `flattened` field type. {issue}24354[24354] {pull}24636[24636] - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] +- Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index fffec3d17de..2071517c877 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -17,6 +17,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2318,6 +2319,26 @@ type: keyword -- +[[exported-fields-awsfargate]] +== AWS Fargate fields + +Module for collecting container logs from Amazon ECS Fargate. + + + +[float] +=== awsfargate + +Fields from Amazon ECS Fargate logs. + + + +[float] +=== log + +Fields for Amazon Fargate container logs. + + [[exported-fields-azure]] == Azure fields diff --git a/filebeat/docs/modules/awsfargate.asciidoc b/filebeat/docs/modules/awsfargate.asciidoc new file mode 100644 index 00000000000..3921e2bbcba --- /dev/null +++ b/filebeat/docs/modules/awsfargate.asciidoc @@ -0,0 +1,139 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-awsfargate]] +[role="xpack"] + +:libbeat-xpack-dir: ../../../x-pack/libbeat + +:modulename: awsfargate +:has-dashboards: false + +== AWS Fargate module + +beta[] + +This module can be used to collect container logs from Amazon ECS on Fargate. +It uses filebeat `awscloudwatch` input to get log files from one or more log +streams in AWS CloudWatch. Logs from all containers in Fargate launch type tasks +can be sent to CloudWatch by adding the `awslogs` log driver under `logConfiguration` +section in the task definition. For example, `logConfiguration` can be added into +the task definition by adding this section into the `containerDefinitions`: + +[source,json] +---- +{ + "logDriver":"awslogs", + "options":{ + "awslogs-group":"awslogs-wordpress", + "awslogs-region":"us-west-2", + "awslogs-stream-prefix":"awslogs-example" + } +} +---- + +The `awsfargate` module requires AWS credentials configuration in order to make AWS API calls. +Users can either use `access_key_id`, `secret_access_key` and/or +`session_token`, or use `role_arn` AWS IAM role, or use shared AWS credentials file. + +Please see <> for more details. + +[float] +=== Module configuration + +Example config: + +[source,yaml] +---- +- module: aws + fargate: + enabled: true + var.credential_profile_name: test-filebeat + var.log_group_arn: arn:aws:logs:us-east-1:1234567890:log-group:/ecs/test-log-group:* +---- + +*`var.log_group_arn`*:: + +ARN of the log group to collect logs from. + +*`var.log_group_name`*:: + +Name of the log group to collect logs from. Note: region_name is required when +log_group_name is given. + +*`var.region_name`*:: + +Region that the specified log group belongs to. + +*`var.log_streams`*:: + +A list of strings of log streams names that Filebeat collect log events from. + +*`var.log_stream_prefix`*:: + +A string to filter the results to include only log events from log streams +that have names starting with this prefix. + +*`var.start_position`*:: + +`start_position` allows user to specify if this input should read log files from +the `beginning` or from the `end`. + +* `beginning`: reads from the beginning of the log group (default). +* `end`: read only new messages from current time minus `scan_frequency` going forward + +*`var.scan_frequency`*:: + +This config parameter sets how often Filebeat checks for new log events from the +specified log group. Default `scan_frequency` is 1 minute, which means Filebeat +will sleep for 1 minute before querying for new logs again. + +*`var.api_timeout`*:: + +The maximum duration of AWS API can take. If it exceeds the timeout, AWS API +will be interrupted. The default AWS API timeout for a message is 120 seconds. +The minimum is 0 seconds. + +*`var.api_sleep`*:: + +This is used to sleep between AWS `FilterLogEvents` API calls inside the same +collection period. `FilterLogEvents` API has a quota of 5 transactions per +second (TPS)/account/Region. By default, `api_sleep` is 200 ms. This value should +only be adjusted when there are multiple Filebeats or multiple Filebeat inputs +collecting logs from the same region and AWS account. + +*`var.shared_credential_file`*:: + +Filename of AWS credential file. + +*`var.credential_profile_name`*:: + +AWS credential profile name. + +*`var.access_key_id`*:: +First part of access key. + +*`var.secret_access_key`*:: +Second part of access key. + +*`var.session_token`*:: +Required when using temporary security credentials. + +*`var.role_arn`*:: +AWS IAM Role to assume. + +*`var.endpoint`*:: + +The custom endpoint used to access AWS APIs. + +[id="awsfargate-credentials"] +include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index 3923c7f8227..3e554cc6407 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -7,6 +7,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -80,6 +81,7 @@ include::modules/activemq.asciidoc[] include::modules/apache.asciidoc[] include::modules/auditd.asciidoc[] include::modules/aws.asciidoc[] +include::modules/awsfargate.asciidoc[] include::modules/azure.asciidoc[] include::modules/barracuda.asciidoc[] include::modules/bluecoat.asciidoc[] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index c44550e351a..5d0dbc7783f 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -274,7 +274,8 @@ def clean_keys(obj): "threatintel.abuseurl", "threatintel.abusemalware", "threatintel.anomali", - "snyk.vulnerabilities" + "snyk.vulnerabilities", + "awsfargate.log" } # dataset + log file pairs for which @timestamp is kept as an exception from above remove_timestamp_exception = { diff --git a/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc index b66d76281c9..d39ec550868 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc @@ -43,10 +43,12 @@ The `aws-cloudwatch` input supports the following configuration options plus the ==== `log_group_arn` ARN of the log group to collect logs from. +[float] ==== `log_group_name` Name of the log group to collect logs from. Note: region_name is required when log_group_name is given. +[float] ==== `region_name` Region that the specified log group belongs to. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d9cc6d6cc9c..f9eea56b2a2 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -348,6 +348,121 @@ filebeat.modules: # The maximum number of messages to return from SQS. Valid values: 1 to 10. #var.max_number_of_messages: 5 +- module: aws + fargate: + enabled: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # ARN of the log group to collect logs from + #var.log_group_arn: arn:aws:logs:us-east-1:123456789012:log-group:test:* + + # Name of the log group to collect logs from + #var.log_group_name: /ecs/test-log-group + + # Region that the specified log group belongs to + #var.region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from + #var.log_streams: ["/ecs/test-log-group/test-log-stream"] + + # A string to filter the results to include only log events from log streams that have names starting with this prefix + #var.log_stream_prefix: /ecs/test-log-group/ + + # Specify if Filebeat should read log files from the beginning or from the end + # Default start_position is beginning + #var.start_position: beginning + + # How often Filebeat checks for new log events from the specified log group + # Default scan_frequency is 1 minute + #var.scan_frequency: 1m + + # Time used to sleep between AWS FilterLogEvents API calls inside the same collection period + # Default api_sleep is 200 ms + #var.api_sleep: 200ms + +#----------------------------- AWS Fargate Module ----------------------------- +- module: awsfargate + log: + enabled: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # ARN of the log group to collect logs from + #var.log_group_arn: arn:aws:logs:us-east-1:123456789012:log-group:test:* + + # Name of the log group to collect logs from + #var.log_group_name: /ecs/test-log-group + + # Region that the specified log group belongs to + #var.region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from + #var.log_streams: ["/ecs/test-log-group/test-log-stream"] + + # A string to filter the results to include only log events from log streams that have names starting with this prefix + #var.log_stream_prefix: /ecs/test-log-group/ + + # Specify if Filebeat should read log files from the beginning or from the end + # Default start_position is beginning + #var.start_position: beginning + + # How often Filebeat checks for new log events from the specified log group + # Default scan_frequency is 1 minute + #var.scan_frequency: 1m + + # Time used to sleep between AWS FilterLogEvents API calls inside the same collection period + # Default api_sleep is 200 ms + #var.api_sleep: 200ms + #-------------------------------- Azure Module -------------------------------- - module: azure # All logs diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index f731ee24a81..94340a8c11c 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -15,6 +15,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/aws" + _ "github.com/elastic/beats/v7/x-pack/filebeat/module/awsfargate" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/azure" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/barracuda" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/bluecoat" diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index e4b521e467f..881138ed9f4 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -250,3 +250,62 @@ # The maximum number of messages to return from SQS. Valid values: 1 to 10. #var.max_number_of_messages: 5 + +- module: aws + fargate: + enabled: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # ARN of the log group to collect logs from + #var.log_group_arn: arn:aws:logs:us-east-1:123456789012:log-group:test:* + + # Name of the log group to collect logs from + #var.log_group_name: /ecs/test-log-group + + # Region that the specified log group belongs to + #var.region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from + #var.log_streams: ["/ecs/test-log-group/test-log-stream"] + + # A string to filter the results to include only log events from log streams that have names starting with this prefix + #var.log_stream_prefix: /ecs/test-log-group/ + + # Specify if Filebeat should read log files from the beginning or from the end + # Default start_position is beginning + #var.start_position: beginning + + # How often Filebeat checks for new log events from the specified log group + # Default scan_frequency is 1 minute + #var.scan_frequency: 1m + + # Time used to sleep between AWS FilterLogEvents API calls inside the same collection period + # Default api_sleep is 200 ms + #var.api_sleep: 200ms diff --git a/x-pack/filebeat/module/awsfargate/_meta/config.yml b/x-pack/filebeat/module/awsfargate/_meta/config.yml new file mode 100644 index 00000000000..2318b322e9a --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/_meta/config.yml @@ -0,0 +1,54 @@ +- module: awsfargate + log: + enabled: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # ARN of the log group to collect logs from + #var.log_group_arn: arn:aws:logs:us-east-1:123456789012:log-group:test:* + + # Name of the log group to collect logs from + #var.log_group_name: /ecs/test-log-group + + # Region that the specified log group belongs to + #var.region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from + #var.log_streams: ["/ecs/test-log-group/test-log-stream"] + + # A string to filter the results to include only log events from log streams that have names starting with this prefix + #var.log_stream_prefix: /ecs/test-log-group/ + + # Specify if Filebeat should read log files from the beginning or from the end + # Default start_position is beginning + #var.start_position: beginning + + # How often Filebeat checks for new log events from the specified log group + # Default scan_frequency is 1 minute + #var.scan_frequency: 1m + + # Time used to sleep between AWS FilterLogEvents API calls inside the same collection period + # Default api_sleep is 200 ms + #var.api_sleep: 200ms diff --git a/x-pack/filebeat/module/awsfargate/_meta/docs.asciidoc b/x-pack/filebeat/module/awsfargate/_meta/docs.asciidoc new file mode 100644 index 00000000000..c17664a5ba6 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/_meta/docs.asciidoc @@ -0,0 +1,126 @@ +[role="xpack"] + +:libbeat-xpack-dir: ../../../x-pack/libbeat + +:modulename: awsfargate +:has-dashboards: false + +== AWS Fargate module + +beta[] + +This module can be used to collect container logs from Amazon ECS on Fargate. +It uses filebeat `awscloudwatch` input to get log files from one or more log +streams in AWS CloudWatch. Logs from all containers in Fargate launch type tasks +can be sent to CloudWatch by adding the `awslogs` log driver under `logConfiguration` +section in the task definition. For example, `logConfiguration` can be added into +the task definition by adding this section into the `containerDefinitions`: + +[source,json] +---- +{ + "logDriver":"awslogs", + "options":{ + "awslogs-group":"awslogs-wordpress", + "awslogs-region":"us-west-2", + "awslogs-stream-prefix":"awslogs-example" + } +} +---- + +The `awsfargate` module requires AWS credentials configuration in order to make AWS API calls. +Users can either use `access_key_id`, `secret_access_key` and/or +`session_token`, or use `role_arn` AWS IAM role, or use shared AWS credentials file. + +Please see <> for more details. + +[float] +=== Module configuration + +Example config: + +[source,yaml] +---- +- module: aws + fargate: + enabled: true + var.credential_profile_name: test-filebeat + var.log_group_arn: arn:aws:logs:us-east-1:1234567890:log-group:/ecs/test-log-group:* +---- + +*`var.log_group_arn`*:: + +ARN of the log group to collect logs from. + +*`var.log_group_name`*:: + +Name of the log group to collect logs from. Note: region_name is required when +log_group_name is given. + +*`var.region_name`*:: + +Region that the specified log group belongs to. + +*`var.log_streams`*:: + +A list of strings of log streams names that Filebeat collect log events from. + +*`var.log_stream_prefix`*:: + +A string to filter the results to include only log events from log streams +that have names starting with this prefix. + +*`var.start_position`*:: + +`start_position` allows user to specify if this input should read log files from +the `beginning` or from the `end`. + +* `beginning`: reads from the beginning of the log group (default). +* `end`: read only new messages from current time minus `scan_frequency` going forward + +*`var.scan_frequency`*:: + +This config parameter sets how often Filebeat checks for new log events from the +specified log group. Default `scan_frequency` is 1 minute, which means Filebeat +will sleep for 1 minute before querying for new logs again. + +*`var.api_timeout`*:: + +The maximum duration of AWS API can take. If it exceeds the timeout, AWS API +will be interrupted. The default AWS API timeout for a message is 120 seconds. +The minimum is 0 seconds. + +*`var.api_sleep`*:: + +This is used to sleep between AWS `FilterLogEvents` API calls inside the same +collection period. `FilterLogEvents` API has a quota of 5 transactions per +second (TPS)/account/Region. By default, `api_sleep` is 200 ms. This value should +only be adjusted when there are multiple Filebeats or multiple Filebeat inputs +collecting logs from the same region and AWS account. + +*`var.shared_credential_file`*:: + +Filename of AWS credential file. + +*`var.credential_profile_name`*:: + +AWS credential profile name. + +*`var.access_key_id`*:: +First part of access key. + +*`var.secret_access_key`*:: +Second part of access key. + +*`var.session_token`*:: +Required when using temporary security credentials. + +*`var.role_arn`*:: +AWS IAM Role to assume. + +*`var.endpoint`*:: + +The custom endpoint used to access AWS APIs. + +[id="awsfargate-credentials"] +include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] diff --git a/x-pack/filebeat/module/awsfargate/_meta/fields.yml b/x-pack/filebeat/module/awsfargate/_meta/fields.yml new file mode 100644 index 00000000000..d5aa527be7a --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: awsfargate + title: AWS Fargate + release: beta + description: > + Module for collecting container logs from Amazon ECS Fargate. + fields: + - name: awsfargate + type: group + description: > + Fields from Amazon ECS Fargate logs. + fields: diff --git a/x-pack/filebeat/module/awsfargate/fields.go b/x-pack/filebeat/module/awsfargate/fields.go new file mode 100644 index 00000000000..43d251cd8cd --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package awsfargate + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "awsfargate", asset.ModuleFieldsPri, AssetAwsfargate); err != nil { + panic(err) + } +} + +// AssetAwsfargate returns asset data. +// This is the base64 encoded gzipped contents of module/awsfargate. +func AssetAwsfargate() string { + return "eJx0kDFuwzAMRXef4l8gPoCGAkHRbJ06dCxYmxKE0qJB0SjS0xc2YsMJnD9S0n9PPOGHrwH0WyNZIucG8OzCAefPD1y2obEwVQ74ZqcG6Ll2lkfPWgJeGgB4134SRlRDpyLceS4JnRanXNggmiqi6YDzQH9a8Pa6EdoGiJmlr2HpOqHQwA9ic/w6ckAyncbb5MBkzmVpe8ZbZNrb5T14DxdN2+yIPOdxL2t6jjSJfy3dAZGk8t3xofReXG31Xp3vV9nuXq0/+A8AAP//VOqIaQ==" +} diff --git a/x-pack/filebeat/module/awsfargate/log/_meta/fields.yml b/x-pack/filebeat/module/awsfargate/log/_meta/fields.yml new file mode 100644 index 00000000000..6fb875c3b73 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: log + type: group + release: beta + default_field: false + description: > + Fields for Amazon Fargate container logs. + fields: diff --git a/x-pack/filebeat/module/awsfargate/log/config/aws-cloudwatch.yml b/x-pack/filebeat/module/awsfargate/log/config/aws-cloudwatch.yml new file mode 100644 index 00000000000..e3908dc6299 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/config/aws-cloudwatch.yml @@ -0,0 +1,63 @@ +type: aws-cloudwatch + +{{ if .log_group_arn }} +log_group_arn: {{ .log_group_arn }} +{{ end }} + +{{ if .log_group_name }} +log_group_name: {{ .log_group_name }} +{{ end }} + +{{ if .region_name }} +region_name: {{ .region_name }} +{{ end }} + +{{ if .start_position }} +start_position: {{ .start_position }} +{{ end }} + +{{ if .scan_frequency }} +scan_frequency: {{ .scan_frequency }} +{{ end }} + +{{ if .api_timeout }} +api_timeout: {{ .api_timeout }} +{{ end }} + +{{ if .api_sleep }} +api_sleep: {{ .api_sleep }} +{{ end }} + +{{ if .credential_profile_name }} +credential_profile_name: {{ .credential_profile_name }} +{{ end }} + +{{ if .shared_credential_file }} +shared_credential_file: {{ .shared_credential_file }} +{{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} + +{{ if .access_key_id }} +access_key_id: {{ .access_key_id }} +{{ end }} + +{{ if .secret_access_key }} +secret_access_key: {{ .secret_access_key }} +{{ end }} + +{{ if .session_token }} +session_token: {{ .session_token }} +{{ end }} + +{{ if .role_arn }} +role_arn: {{ .role_arn }} +{{ end }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.9.0 diff --git a/x-pack/filebeat/module/awsfargate/log/config/file.yml b/x-pack/filebeat/module/awsfargate/log/config/file.yml new file mode 100644 index 00000000000..df0f9450c17 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/config/file.yml @@ -0,0 +1,11 @@ +type: log +paths: + {{ range $i, $path := .paths }} + - {{$path}} + {{ end }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.9.0 diff --git a/x-pack/filebeat/module/awsfargate/log/ingest/pipeline.yml b/x-pack/filebeat/module/awsfargate/log/ingest/pipeline.yml new file mode 100644 index 00000000000..f0b587752e9 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/ingest/pipeline.yml @@ -0,0 +1,11 @@ +description: "Pipeline for AWS Fargate logs" + +processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + +on_failure: + - set: + field: "error.message" + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/awsfargate/log/manifest.yml b/x-pack/filebeat/module/awsfargate/log/manifest.yml new file mode 100644 index 00000000000..ca5fb61d056 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/manifest.yml @@ -0,0 +1,25 @@ +module_version: 1.0 + +var: + - name: input + default: aws-cloudwatch + - name: shared_credential_file + - name: credential_profile_name + - name: endpoint + - name: access_key_id + - name: secret_access_key + - name: session_token + - name: role_arn + - name: log_group_arn + - name: log_group_name + - name: region_name + - name: log_streams + - name: log_stream_prefix + - name: start_position + default: beginning + - name: scan_frequency + - name: api_timeout + - name: api_sleep + +ingest_pipeline: ingest/pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/awsfargate/log/test/nginx.log b/x-pack/filebeat/module/awsfargate/log/test/nginx.log new file mode 100644 index 00000000000..f255fb92962 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/test/nginx.log @@ -0,0 +1,2 @@ +10.0.1.195 - - [24/Sep/2020:16:45:00 +0000] "GET /config/getuser?index=0 HTTP/1.1" 404 153 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0" "45.148.10.28" +10.0.1.195 - - [24/Sep/2020:16:45:19 +0000] "GET / HTTP/1.1" 200 612 "-" "ELB-HealthChecker/2.0" "-" diff --git a/x-pack/filebeat/module/awsfargate/log/test/nginx.log-expected.json b/x-pack/filebeat/module/awsfargate/log/test/nginx.log-expected.json new file mode 100644 index 00000000000..35c0e294776 --- /dev/null +++ b/x-pack/filebeat/module/awsfargate/log/test/nginx.log-expected.json @@ -0,0 +1,20 @@ +[ + { + "event.dataset": "awsfargate.log", + "event.module": "awsfargate", + "fileset.name": "log", + "input.type": "log", + "log.offset": 0, + "message": "10.0.1.195 - - [24/Sep/2020:16:45:00 +0000] \"GET /config/getuser?index=0 HTTP/1.1\" 404 153 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0\" \"45.148.10.28\"", + "service.type": "awsfargate" + }, + { + "event.dataset": "awsfargate.log", + "event.module": "awsfargate", + "fileset.name": "log", + "input.type": "log", + "log.offset": 189, + "message": "10.0.1.195 - - [24/Sep/2020:16:45:19 +0000] \"GET / HTTP/1.1\" 200 612 \"-\" \"ELB-HealthChecker/2.0\" \"-\"", + "service.type": "awsfargate" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json index e39b9b0013a..cc22f1f1e16 100644 --- a/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/intel/test/intel-json.log-expected.json @@ -2,8 +2,8 @@ { "@timestamp": "2019-11-06T09:03:00.989Z", "destination.address": "198.41.0.4", - "destination.as.number": 10515, - "destination.as.organization.name": "CLT-NIC", + "destination.as.number": 20172, + "destination.as.organization.name": "VGRS-AC27", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.country_name": "United States", diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index f3d2ac1f7c9..b79a4212c93 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -253,3 +253,62 @@ # The maximum number of messages to return from SQS. Valid values: 1 to 10. #var.max_number_of_messages: 5 + +- module: aws + fargate: + enabled: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # ARN of the log group to collect logs from + #var.log_group_arn: arn:aws:logs:us-east-1:123456789012:log-group:test:* + + # Name of the log group to collect logs from + #var.log_group_name: /ecs/test-log-group + + # Region that the specified log group belongs to + #var.region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from + #var.log_streams: ["/ecs/test-log-group/test-log-stream"] + + # A string to filter the results to include only log events from log streams that have names starting with this prefix + #var.log_stream_prefix: /ecs/test-log-group/ + + # Specify if Filebeat should read log files from the beginning or from the end + # Default start_position is beginning + #var.start_position: beginning + + # How often Filebeat checks for new log events from the specified log group + # Default scan_frequency is 1 minute + #var.scan_frequency: 1m + + # Time used to sleep between AWS FilterLogEvents API calls inside the same collection period + # Default api_sleep is 200 ms + #var.api_sleep: 200ms diff --git a/x-pack/filebeat/modules.d/awsfargate.yml.disabled b/x-pack/filebeat/modules.d/awsfargate.yml.disabled new file mode 100644 index 00000000000..225892f7fbe --- /dev/null +++ b/x-pack/filebeat/modules.d/awsfargate.yml.disabled @@ -0,0 +1,57 @@ +# Module: awsfargate +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-awsfargate.html + +- module: awsfargate + log: + enabled: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # ARN of the log group to collect logs from + #var.log_group_arn: arn:aws:logs:us-east-1:123456789012:log-group:test:* + + # Name of the log group to collect logs from + #var.log_group_name: /ecs/test-log-group + + # Region that the specified log group belongs to + #var.region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from + #var.log_streams: ["/ecs/test-log-group/test-log-stream"] + + # A string to filter the results to include only log events from log streams that have names starting with this prefix + #var.log_stream_prefix: /ecs/test-log-group/ + + # Specify if Filebeat should read log files from the beginning or from the end + # Default start_position is beginning + #var.start_position: beginning + + # How often Filebeat checks for new log events from the specified log group + # Default scan_frequency is 1 minute + #var.scan_frequency: 1m + + # Time used to sleep between AWS FilterLogEvents API calls inside the same collection period + # Default api_sleep is 200 ms + #var.api_sleep: 200ms