From 26cc4ad49a47f0b509053ea896406b324cd98270 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 12 May 2020 21:20:23 -0400 Subject: [PATCH] Support application default credentials (ADC) for Google Pub/Sub (#15668) (#17791) Update the Google Pub/Sub input to support reading Application Default Credentials (ADC) in addition to the credentials_file and credentials_json config options. If neither config option is set then it will attempt to search for the default credentials. Generally this means reading the GOOGLE_APPLICATION_CREDENTIALS environment variable plus search a few other well known locations. The googlecloud module was updates to support all three authentication mechanisms. Co-authored-by: Michal Wasilewski Co-authored-by: Andrew Kroh (cherry picked from commit 058de35affcc23f87d7eecb5ca24bf8212cfcaed) Co-authored-by: Michal --- CHANGELOG.next.asciidoc | 1 + .../docs/inputs/input-google-pubsub.asciidoc | 9 +++-- x-pack/filebeat/input/googlepubsub/config.go | 31 ++++++++++++++--- .../input/googlepubsub/config_test.go | 34 +++++++++++++++++++ .../input/googlepubsub/pubsub_test.go | 2 +- .../input/googlepubsub/testdata/fake.json | 12 +++++++ .../module/googlecloud/audit/config/input.yml | 5 +++ .../module/googlecloud/audit/manifest.yml | 2 +- .../googlecloud/firewall/config/input.yml | 5 +++ .../module/googlecloud/firewall/manifest.yml | 2 +- .../googlecloud/vpcflow/config/input.yml | 5 +++ .../module/googlecloud/vpcflow/manifest.yml | 2 +- 12 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 x-pack/filebeat/input/googlepubsub/config_test.go create mode 100644 x-pack/filebeat/input/googlepubsub/testdata/fake.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6633e9263ea..54f93f0f284 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -254,6 +254,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155] - Release Google Cloud module as GA. {pull}17511[17511] - Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] +- Add support for Google Application Default Credentials to the Google Pub/Sub input and Google Cloud modules. {pull}15668[15668] - Make `decode_cef` processor GA. {pull}17944[17944] - Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] diff --git a/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc b/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc index 30bc1fb25f7..c03a3327602 100644 --- a/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc @@ -79,15 +79,18 @@ unprocessed messages. Default is 1000. ==== `credentials_file` Path to a JSON file containing the credentials and key used to subscribe. -One credential option must be set. +As an alternative you can use the `credentials_json` config option or rely on +https://cloud.google.com/docs/authentication/production[Google Application +Default Credentials] (ADC). [float] ==== `credentials_json` JSON blob containing the credentials and key used to subscribe. This can be as an alternative to `credentials_file` if you want to embed the credential data -within your config file or put the information into a keystore. One credential -option must be set. +within your config file or put the information into a keystore. You may also use +https://cloud.google.com/docs/authentication/production[Google Application +Default Credentials] (ADC). [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/input/googlepubsub/config.go b/x-pack/filebeat/input/googlepubsub/config.go index c0686d46510..56f02f3a471 100644 --- a/x-pack/filebeat/input/googlepubsub/config.go +++ b/x-pack/filebeat/input/googlepubsub/config.go @@ -5,7 +5,12 @@ package googlepubsub import ( - "github.com/pkg/errors" + "context" + "fmt" + "os" + + "cloud.google.com/go/pubsub" + "golang.org/x/oauth2/google" ) type config struct { @@ -31,10 +36,28 @@ type config struct { } func (c *config) Validate() error { - if c.CredentialsFile == "" && len(c.CredentialsJSON) == 0 { - return errors.New("credentials_file or credentials_json is required for pubsub input") + // credentials_file + if c.CredentialsFile != "" { + if _, err := os.Stat(c.CredentialsFile); os.IsNotExist(err) { + return fmt.Errorf("credentials_file is configured, but the file %q cannot be found", c.CredentialsFile) + } else { + return nil + } + } + + // credentials_json + if len(c.CredentialsJSON) > 0 { + return nil } - return nil + + // Application Default Credentials (ADC) + ctx := context.Background() + if _, err := google.FindDefaultCredentials(ctx, pubsub.ScopePubSub); err == nil { + return nil + } + + return fmt.Errorf("no authentication credentials were configured or detected " + + "(credentials_file, credentials_json, and application default credentials (ADC))") } func defaultConfig() config { diff --git a/x-pack/filebeat/input/googlepubsub/config_test.go b/x-pack/filebeat/input/googlepubsub/config_test.go new file mode 100644 index 00000000000..6e949326ca3 --- /dev/null +++ b/x-pack/filebeat/input/googlepubsub/config_test.go @@ -0,0 +1,34 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package googlepubsub + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +const googleApplicationCredentialsVar = "GOOGLE_APPLICATION_CREDENTIALS" + +func TestConfigValidateGoogleAppDefaultCreds(t *testing.T) { + // Return the environment variables to their original state. + original, found := os.LookupEnv(googleApplicationCredentialsVar) + defer func() { + if found { + os.Setenv(googleApplicationCredentialsVar, original) + } else { + os.Unsetenv(googleApplicationCredentialsVar) + } + }() + + // Validate that it finds the application default credentials and does + // not trigger a config validation error because credentials were not + // set in the config. + os.Setenv(googleApplicationCredentialsVar, filepath.Clean("testdata/fake.json")) + c := defaultConfig() + assert.NoError(t, c.Validate()) +} diff --git a/x-pack/filebeat/input/googlepubsub/pubsub_test.go b/x-pack/filebeat/input/googlepubsub/pubsub_test.go index 17005ac57f3..17863862c8a 100644 --- a/x-pack/filebeat/input/googlepubsub/pubsub_test.go +++ b/x-pack/filebeat/input/googlepubsub/pubsub_test.go @@ -203,7 +203,7 @@ func defaultTestConfig() *common.Config { "name": emulatorSubscription, "create": true, }, - "credentials_file": "NONE FOR EMULATOR TESTING", + "credentials_file": "testdata/fake.json", }) } diff --git a/x-pack/filebeat/input/googlepubsub/testdata/fake.json b/x-pack/filebeat/input/googlepubsub/testdata/fake.json new file mode 100644 index 00000000000..62bc9a26633 --- /dev/null +++ b/x-pack/filebeat/input/googlepubsub/testdata/fake.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "private_key": "", + "client_email": "foo@bar.com", + "client_id": "0", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://foo.bar/path" +} diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml index 0cbf32d8943..3cc0edf9f1c 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/audit/config/input.yml @@ -4,7 +4,12 @@ type: google-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} +{{ if .credentials_file }} credentials_file: {{ .credentials_file }} +{{ end }} +{{ if .credentials_json }} +credentials_json: {{ .credentials_json }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/googlecloud/audit/manifest.yml b/x-pack/filebeat/module/googlecloud/audit/manifest.yml index 3f815e1370a..347d8eaa1cb 100644 --- a/x-pack/filebeat/module/googlecloud/audit/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/audit/manifest.yml @@ -10,7 +10,7 @@ var: - name: subscription_name default: filebeat-googlecloud-audit - name: credentials_file - default: googlecloud-audit-reader-service-identity.json + - name: credentials_json - name: keep_original_message default: false ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml index dd617f8d288..377223630e8 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml @@ -4,7 +4,12 @@ type: google-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} +{{ if .credentials_file }} credentials_file: {{ .credentials_file }} +{{ end }} +{{ if .credentials_json }} +credentials_json: {{ .credentials_json }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml index ec265f97712..53e4c5dc69d 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml @@ -10,7 +10,7 @@ var: - name: subscription_name default: filebeat-googlecloud-firewall - name: credentials_file - default: googlecloud-firewall-reader-service-identity.json + - name: credentials_json - name: debug default: false - name: keep_original_message diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml index b8b7a260bf4..3de9c7dd28f 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml @@ -4,7 +4,12 @@ type: google-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} +{{ if .credentials_file }} credentials_file: {{ .credentials_file }} +{{ end }} +{{ if .credentials_json }} +credentials_json: {{ .credentials_json }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml b/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml index a84b56a3150..6c2ec7c1da3 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml @@ -10,7 +10,7 @@ var: - name: subscription_name default: filebeat-googlecloud-vpcflow - name: credentials_file - default: googlecloud-vpcflow-reader-service-identity.json + - name: credentials_json - name: keep_original_message default: false ingest_pipeline: ingest/pipeline.yml