Skip to content

Commit

Permalink
[Filebeat] Add support for specifying AWS cred file (#15656) (#15746)
Browse files Browse the repository at this point in the history
* Add optional AWS shared_credential_file to all s3 input modules
* Made AWS credential_profile_name optional for all s3 input modules

Fixes #15652

(cherry picked from commit 005f474)
  • Loading branch information
leehinman authored Jan 22, 2020
1 parent 446b07f commit 9c5cb89
Show file tree
Hide file tree
Showing 14 changed files with 175 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- netflow: Fix bytes/packets counters on some devices (NSEL and Netstream). {pull}15449[15449]
- netflow: Fix compatibility with some Cisco devices by changing the field `class_id` from short to long. {pull}15449[15449]
- Fixed dashboard for Cisco ASA Firewall. {issue}15420[15420] {pull}15553[15553]
- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]

*Heartbeat*

Expand Down
36 changes: 32 additions & 4 deletions filebeat/docs/modules/aws.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -45,41 +45,69 @@ Example config:
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
elb:
enabled: false
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
vpcflow:
enabled: false
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
cloudtrail:
enabled: false
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
----

*`var.queue_url`*::

AWS SQS queue url.

*`var.shared_credential_file`*::

Filename of AWS credential file.

*`var.credential_profile_name`*::

AWS credential profile name.
Expand Down
24 changes: 24 additions & 0 deletions x-pack/filebeat/filebeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,13 @@ filebeat.modules:
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

elb:
Expand All @@ -111,7 +117,13 @@ filebeat.modules:
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

vpcflow:
Expand All @@ -120,7 +132,13 @@ filebeat.modules:
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

cloudtrail:
Expand All @@ -129,7 +147,13 @@ filebeat.modules:
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

#-------------------------------- Azure Module --------------------------------
Expand Down
24 changes: 24 additions & 0 deletions x-pack/filebeat/module/aws/_meta/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,13 @@
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

elb:
Expand All @@ -14,7 +20,13 @@
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

vpcflow:
Expand All @@ -23,7 +35,13 @@
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws

cloudtrail:
Expand All @@ -32,5 +50,11 @@
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue

# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
#var.shared_credential_file: /etc/filebeat/aws_credentials

# Profile name for aws credential
# If not set the default profile is used
#var.credential_profile_name: fb-aws
36 changes: 32 additions & 4 deletions x-pack/filebeat/module/aws/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -40,41 +40,69 @@ Example config:
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
elb:
enabled: false
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
vpcflow:
enabled: false
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
cloudtrail:
enabled: false
# AWS SQS queue url
#var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
# Filename of AWS credential file
# If not set "$HOME/.aws/credentials" is used on Linux/Mac
# "%UserProfile%\.aws\credentials" is used on Windows
# var.shared_credential_file: /etc/filebeat/aws_credentials
# Profile name for aws credential
#var.credential_profile_name: fb-aws
# If not set the default profile is used
# var.credential_profile_name: fb-aws
----

*`var.queue_url`*::

AWS SQS queue url.

*`var.shared_credential_file`*::

Filename of AWS credential file.

*`var.credential_profile_name`*::

AWS credential profile name.
Expand Down
10 changes: 9 additions & 1 deletion x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,17 @@

type: s3
queue_url: {{ .queue_url }}
credential_profile_name: {{ .credential_profile_name }}
expand_event_list_from_field: Records

{{ if .credential_profile_name }}
credential_profile_name: {{ .credential_profile_name }}
{{ end }}

{{ if .shared_credential_file }}
shared_credential_file: {{ .shared_credential_file }}
{{ end }}


{{ else if eq .input "file" }}

type: log
Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/aws/cloudtrail/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ module_version: 1.0
var:
- name: input
default: s3
- name: shared_credential_file
- name: credential_profile_name

ingest_pipeline: ingest/pipeline.yml
input: config/cloudtrail.yml
7 changes: 7 additions & 0 deletions x-pack/filebeat/module/aws/elb/config/s3.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
type: s3
queue_url: {{ .queue_url }}

{{ if .credential_profile_name }}
credential_profile_name: {{ .credential_profile_name }}
{{ end }}

{{ if .shared_credential_file }}
shared_credential_file: {{ .shared_credential_file }}
{{ end }}
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/aws/elb/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ module_version: 1.0
var:
- name: input
default: s3
- name: shared_credential_file
- name: credential_profile_name

ingest_pipeline: ingest/pipeline.yml
input: config/{{.input}}.yml
Expand Down
7 changes: 7 additions & 0 deletions x-pack/filebeat/module/aws/s3access/config/s3.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
type: s3
queue_url: {{ .queue_url }}

{{ if .credential_profile_name }}
credential_profile_name: {{ .credential_profile_name }}
{{ end }}

{{ if .shared_credential_file }}
shared_credential_file: {{ .shared_credential_file }}
{{ end }}
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/aws/s3access/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ module_version: 1.0
var:
- name: input
default: s3
- name: shared_credential_file
- name: credential_profile_name

ingest_pipeline: ingest/pipeline.yml
input: config/{{.input}}.yml
7 changes: 7 additions & 0 deletions x-pack/filebeat/module/aws/vpcflow/config/input.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,14 @@

type: s3
queue_url: {{ .queue_url }}

{{ if .credential_profile_name }}
credential_profile_name: {{ .credential_profile_name }}
{{ end }}

{{ if .shared_credential_file }}
shared_credential_file: {{ .shared_credential_file }}
{{ end }}

{{ else if eq .input "file" }}

Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/aws/vpcflow/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ module_version: 1.0
var:
- name: input
default: s3
- name: shared_credential_file
- name: credential_profile_name

ingest_pipeline: ingest/pipeline.yml
input: config/input.yml
Loading

0 comments on commit 9c5cb89

Please sign in to comment.