From 25b1631ec35d8d62e7760644e8d54b20af15cf78 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 2 Jun 2020 11:59:53 +0200 Subject: [PATCH 1/9] Filebeat HTTPJSON input initial changes to support oauth2 client_credentials --- x-pack/filebeat/input/httpjson/config.go | 14 +++++ x-pack/filebeat/input/httpjson/input.go | 65 ++++++++++++++++-------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index cb1e12ba417..a96e328eeae 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -17,6 +17,7 @@ import ( // Config contains information about httpjson configuration type config struct { + OAuth2 *OAuth2 `config:"oauth2"` APIKey string `config:"api_key"` AuthenticationScheme string `config:"authentication_scheme"` HTTPClientTimeout time.Duration `config:"http_client_timeout"` @@ -32,6 +33,14 @@ type config struct { URL string `config:"url" validate:"required"` } +// OAuth2 contains information about oauth2 authentication settings. +type OAuth2 struct { + TokenURL string `config:"token_url"` + ClientID string `config:"client_id"` + ClientSecret string `config:"client_secret"` + Scopes []string `config:"scopes"` +} + // Pagination contains information about httpjson pagination settings type Pagination struct { Enabled *bool `config:"enabled"` @@ -84,6 +93,11 @@ func (c *config) Validate() error { } } } + if c.OAuth2 != nil { + if c.APIKey != "" || c.AuthenticationScheme != "" { + return errors.Errorf("invalid configuration: both oauth2 and api_key or authentication_scheme cannot be set simultaneously") + } + } return nil } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index bcb2296c039..f921d51ee6c 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -17,6 +17,8 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" @@ -397,31 +399,11 @@ func (in *HttpjsonInput) run() error { ctx, cancel := context.WithCancel(in.workerCtx) defer cancel() - tlsConfig, err := tlscommon.LoadTLSConfig(in.config.TLS) + client, err := in.newHTTPClient(ctx) if err != nil { return err } - var dialer, tlsDialer transport.Dialer - - dialer = transport.NetDialer(in.config.HTTPClientTimeout) - tlsDialer, err = transport.TLSDialer(dialer, tlsConfig, in.config.HTTPClientTimeout) - if err != nil { - return err - } - - // Make transport client - var client *http.Client - client = &http.Client{ - Transport: &http.Transport{ - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: tlsConfig.ToConfig(), - DisableKeepAlives: true, - }, - Timeout: in.config.HTTPClientTimeout, - } - ri := &RequestInfo{ URL: in.URL, ContentMap: common.MapStr{}, @@ -462,6 +444,47 @@ func (in *HttpjsonInput) Wait() { in.Stop() } +func (in *HttpjsonInput) newHTTPClient(ctx context.Context) (*http.Client, error) { + tlsConfig, err := tlscommon.LoadTLSConfig(in.config.TLS) + if err != nil { + return nil, err + } + + var dialer, tlsDialer transport.Dialer + + dialer = transport.NetDialer(in.config.HTTPClientTimeout) + tlsDialer, err = transport.TLSDialer(dialer, tlsConfig, in.config.HTTPClientTimeout) + if err != nil { + return nil, err + } + + // Make transport client + client := &http.Client{ + Transport: &http.Transport{ + Dial: dialer.Dial, + DialTLS: tlsDialer.Dial, + TLSClientConfig: tlsConfig.ToConfig(), + DisableKeepAlives: true, + }, + Timeout: in.config.HTTPClientTimeout, + } + + if in.config.OAuth2 == nil { + return client, nil + } + + ctx = context.WithValue(ctx, oauth2.HTTPClient, client) + + creds := clientcredentials.Config{ + ClientID: in.config.OAuth2.ClientID, + ClientSecret: in.config.OAuth2.ClientSecret, + TokenURL: in.config.OAuth2.TokenURL, + Scopes: in.config.OAuth2.Scopes, + } + + return creds.Client(ctx), nil +} + func makeEvent(body string) beat.Event { fields := common.MapStr{ "event": common.MapStr{ From 4150b7e47ad80d85fc991be088d956a2a200ca54 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 3 Jun 2020 14:38:30 +0200 Subject: [PATCH 2/9] [Filebeat][httpjson] Add EndpointParams option to oauth config --- x-pack/filebeat/input/httpjson/config.go | 9 +++++---- x-pack/filebeat/input/httpjson/input.go | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index a96e328eeae..47f06d26069 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -35,10 +35,11 @@ type config struct { // OAuth2 contains information about oauth2 authentication settings. type OAuth2 struct { - TokenURL string `config:"token_url"` - ClientID string `config:"client_id"` - ClientSecret string `config:"client_secret"` - Scopes []string `config:"scopes"` + TokenURL string `config:"token_url"` + EndpointParams map[string][]string `config:"endpoint_params"` + ClientID string `config:"client_id"` + ClientSecret string `config:"client_secret"` + Scopes []string `config:"scopes"` } // Pagination contains information about httpjson pagination settings diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index f921d51ee6c..ff4378f2beb 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -474,12 +474,12 @@ func (in *HttpjsonInput) newHTTPClient(ctx context.Context) (*http.Client, error } ctx = context.WithValue(ctx, oauth2.HTTPClient, client) - creds := clientcredentials.Config{ - ClientID: in.config.OAuth2.ClientID, - ClientSecret: in.config.OAuth2.ClientSecret, - TokenURL: in.config.OAuth2.TokenURL, - Scopes: in.config.OAuth2.Scopes, + ClientID: in.config.OAuth2.ClientID, + ClientSecret: in.config.OAuth2.ClientSecret, + TokenURL: in.config.OAuth2.TokenURL, + Scopes: in.config.OAuth2.Scopes, + EndpointParams: in.config.OAuth2.EndpointParams, } return creds.Client(ctx), nil From 488a7f9d1958e025884a5d3f0e6cd84b4632e01b Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 4 Jun 2020 17:35:31 +0200 Subject: [PATCH 3/9] Add provider specific settings to oauth httpjson --- .../x/oauth2/endpoints/endpoints.go | 238 ++++++++++++++++++ vendor/modules.txt | 1 + x-pack/filebeat/input/httpjson/config.go | 16 +- .../filebeat/input/httpjson/config_oauth.go | 161 ++++++++++++ x-pack/filebeat/input/httpjson/input.go | 17 +- 5 files changed, 407 insertions(+), 26 deletions(-) create mode 100644 vendor/golang.org/x/oauth2/endpoints/endpoints.go create mode 100644 x-pack/filebeat/input/httpjson/config_oauth.go diff --git a/vendor/golang.org/x/oauth2/endpoints/endpoints.go b/vendor/golang.org/x/oauth2/endpoints/endpoints.go new file mode 100644 index 00000000000..811e101f920 --- /dev/null +++ b/vendor/golang.org/x/oauth2/endpoints/endpoints.go @@ -0,0 +1,238 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package endpoints provides constants for using OAuth2 to access various services. +package endpoints + +import ( + "strings" + + "golang.org/x/oauth2" +) + +// Amazon is the endpoint for Amazon. +var Amazon = oauth2.Endpoint{ + AuthURL: "https://www.amazon.com/ap/oa", + TokenURL: "https://api.amazon.com/auth/o2/token", +} + +// Bitbucket is the endpoint for Bitbucket. +var Bitbucket = oauth2.Endpoint{ + AuthURL: "https://bitbucket.org/site/oauth2/authorize", + TokenURL: "https://bitbucket.org/site/oauth2/access_token", +} + +// Cern is the endpoint for CERN. +var Cern = oauth2.Endpoint{ + AuthURL: "https://oauth.web.cern.ch/OAuth/Authorize", + TokenURL: "https://oauth.web.cern.ch/OAuth/Token", +} + +// Facebook is the endpoint for Facebook. +var Facebook = oauth2.Endpoint{ + AuthURL: "https://www.facebook.com/v3.2/dialog/oauth", + TokenURL: "https://graph.facebook.com/v3.2/oauth/access_token", +} + +// Foursquare is the endpoint for Foursquare. +var Foursquare = oauth2.Endpoint{ + AuthURL: "https://foursquare.com/oauth2/authorize", + TokenURL: "https://foursquare.com/oauth2/access_token", +} + +// Fitbit is the endpoint for Fitbit. +var Fitbit = oauth2.Endpoint{ + AuthURL: "https://www.fitbit.com/oauth2/authorize", + TokenURL: "https://api.fitbit.com/oauth2/token", +} + +// GitHub is the endpoint for Github. +var GitHub = oauth2.Endpoint{ + AuthURL: "https://github.com/login/oauth/authorize", + TokenURL: "https://github.com/login/oauth/access_token", +} + +// GitLab is the endpoint for GitLab. +var GitLab = oauth2.Endpoint{ + AuthURL: "https://gitlab.com/oauth/authorize", + TokenURL: "https://gitlab.com/oauth/token", +} + +// Google is the endpoint for Google. +var Google = oauth2.Endpoint{ + AuthURL: "https://accounts.google.com/o/oauth2/auth", + TokenURL: "https://oauth2.googleapis.com/token", +} + +// Heroku is the endpoint for Heroku. +var Heroku = oauth2.Endpoint{ + AuthURL: "https://id.heroku.com/oauth/authorize", + TokenURL: "https://id.heroku.com/oauth/token", +} + +// HipChat is the endpoint for HipChat. +var HipChat = oauth2.Endpoint{ + AuthURL: "https://www.hipchat.com/users/authorize", + TokenURL: "https://api.hipchat.com/v2/oauth/token", +} + +// Instagram is the endpoint for Instagram. +var Instagram = oauth2.Endpoint{ + AuthURL: "https://api.instagram.com/oauth/authorize", + TokenURL: "https://api.instagram.com/oauth/access_token", +} + +// KaKao is the endpoint for KaKao. +var KaKao = oauth2.Endpoint{ + AuthURL: "https://kauth.kakao.com/oauth/authorize", + TokenURL: "https://kauth.kakao.com/oauth/token", +} + +// LinkedIn is the endpoint for LinkedIn. +var LinkedIn = oauth2.Endpoint{ + AuthURL: "https://www.linkedin.com/oauth/v2/authorization", + TokenURL: "https://www.linkedin.com/oauth/v2/accessToken", +} + +// Mailchimp is the endpoint for Mailchimp. +var Mailchimp = oauth2.Endpoint{ + AuthURL: "https://login.mailchimp.com/oauth2/authorize", + TokenURL: "https://login.mailchimp.com/oauth2/token", +} + +// Mailru is the endpoint for Mail.Ru. +var Mailru = oauth2.Endpoint{ + AuthURL: "https://o2.mail.ru/login", + TokenURL: "https://o2.mail.ru/token", +} + +// MediaMath is the endpoint for MediaMath. +var MediaMath = oauth2.Endpoint{ + AuthURL: "https://api.mediamath.com/oauth2/v1.0/authorize", + TokenURL: "https://api.mediamath.com/oauth2/v1.0/token", +} + +// MediaMathSandbox is the endpoint for MediaMath Sandbox. +var MediaMathSandbox = oauth2.Endpoint{ + AuthURL: "https://t1sandbox.mediamath.com/oauth2/v1.0/authorize", + TokenURL: "https://t1sandbox.mediamath.com/oauth2/v1.0/token", +} + +// Microsoft is the endpoint for Microsoft. +var Microsoft = oauth2.Endpoint{ + AuthURL: "https://login.live.com/oauth20_authorize.srf", + TokenURL: "https://login.live.com/oauth20_token.srf", +} + +// NokiaHealth is the endpoint for Nokia Health. +var NokiaHealth = oauth2.Endpoint{ + AuthURL: "https://account.health.nokia.com/oauth2_user/authorize2", + TokenURL: "https://account.health.nokia.com/oauth2/token", +} + +// Odnoklassniki is the endpoint for Odnoklassniki. +var Odnoklassniki = oauth2.Endpoint{ + AuthURL: "https://www.odnoklassniki.ru/oauth/authorize", + TokenURL: "https://api.odnoklassniki.ru/oauth/token.do", +} + +// PayPal is the endpoint for PayPal. +var PayPal = oauth2.Endpoint{ + AuthURL: "https://www.paypal.com/webapps/auth/protocol/openidconnect/v1/authorize", + TokenURL: "https://api.paypal.com/v1/identity/openidconnect/tokenservice", +} + +// PayPalSandbox is the endpoint for PayPal Sandbox. +var PayPalSandbox = oauth2.Endpoint{ + AuthURL: "https://www.sandbox.paypal.com/webapps/auth/protocol/openidconnect/v1/authorize", + TokenURL: "https://api.sandbox.paypal.com/v1/identity/openidconnect/tokenservice", +} + +// Slack is the endpoint for Slack. +var Slack = oauth2.Endpoint{ + AuthURL: "https://slack.com/oauth/authorize", + TokenURL: "https://slack.com/api/oauth.access", +} + +// Spotify is the endpoint for Spotify. +var Spotify = oauth2.Endpoint{ + AuthURL: "https://accounts.spotify.com/authorize", + TokenURL: "https://accounts.spotify.com/api/token", +} + +// StackOverflow is the endpoint for Stack Overflow. +var StackOverflow = oauth2.Endpoint{ + AuthURL: "https://stackoverflow.com/oauth", + TokenURL: "https://stackoverflow.com/oauth/access_token", +} + +// Twitch is the endpoint for Twitch. +var Twitch = oauth2.Endpoint{ + AuthURL: "https://id.twitch.tv/oauth2/authorize", + TokenURL: "https://id.twitch.tv/oauth2/token", +} + +// Uber is the endpoint for Uber. +var Uber = oauth2.Endpoint{ + AuthURL: "https://login.uber.com/oauth/v2/authorize", + TokenURL: "https://login.uber.com/oauth/v2/token", +} + +// Vk is the endpoint for Vk. +var Vk = oauth2.Endpoint{ + AuthURL: "https://oauth.vk.com/authorize", + TokenURL: "https://oauth.vk.com/access_token", +} + +// Yahoo is the endpoint for Yahoo. +var Yahoo = oauth2.Endpoint{ + AuthURL: "https://api.login.yahoo.com/oauth2/request_auth", + TokenURL: "https://api.login.yahoo.com/oauth2/get_token", +} + +// Yandex is the endpoint for Yandex. +var Yandex = oauth2.Endpoint{ + AuthURL: "https://oauth.yandex.com/authorize", + TokenURL: "https://oauth.yandex.com/token", +} + +// AzureAD returns a new oauth2.Endpoint for the given tenant at Azure Active Directory. +// If tenant is empty, it uses the tenant called `common`. +// +// For more information see: +// https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-v2-protocols#endpoints +func AzureAD(tenant string) oauth2.Endpoint { + if tenant == "" { + tenant = "common" + } + return oauth2.Endpoint{ + AuthURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/authorize", + TokenURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/token", + } +} + +// HipChatServer returns a new oauth2.Endpoint for a HipChat Server instance +// running on the given domain or host. +func HipChatServer(host string) oauth2.Endpoint { + return oauth2.Endpoint{ + AuthURL: "https://" + host + "/users/authorize", + TokenURL: "https://" + host + "/v2/oauth/token", + } +} + +// AWSCognito returns a new oauth2.Endpoint for the supplied AWS Cognito domain which is +// linked to your Cognito User Pool. +// +// Example domain: https://testing.auth.us-east-1.amazoncognito.com +// +// For more information see: +// https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-user-pools-assign-domain.html +// https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-userpools-server-contract-reference.html +func AWSCognito(domain string) oauth2.Endpoint { + domain = strings.TrimRight(domain, "/") + return oauth2.Endpoint{ + AuthURL: domain + "/oauth2/authorize", + TokenURL: domain + "/oauth2/token", + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a0e9358788f..94f523d32fb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -947,6 +947,7 @@ golang.org/x/net/websocket # golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/oauth2 golang.org/x/oauth2/clientcredentials +golang.org/x/oauth2/endpoints golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 47f06d26069..b3029ef488a 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -33,15 +33,6 @@ type config struct { URL string `config:"url" validate:"required"` } -// OAuth2 contains information about oauth2 authentication settings. -type OAuth2 struct { - TokenURL string `config:"token_url"` - EndpointParams map[string][]string `config:"endpoint_params"` - ClientID string `config:"client_id"` - ClientSecret string `config:"client_secret"` - Scopes []string `config:"scopes"` -} - // Pagination contains information about httpjson pagination settings type Pagination struct { Enabled *bool `config:"enabled"` @@ -72,9 +63,7 @@ type RateLimit struct { func (c *config) Validate() error { switch strings.ToUpper(c.HTTPMethod) { - case "GET": - break - case "POST": + case "GET", "POST": break default: return errors.Errorf("httpjson input: Invalid http_method, %s", c.HTTPMethod) @@ -98,6 +87,9 @@ func (c *config) Validate() error { if c.APIKey != "" || c.AuthenticationScheme != "" { return errors.Errorf("invalid configuration: both oauth2 and api_key or authentication_scheme cannot be set simultaneously") } + if err := c.OAuth2.validate(); err != nil { + return err + } } return nil } diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go new file mode 100644 index 00000000000..f9e1e3f3a8a --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -0,0 +1,161 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + "golang.org/x/oauth2/endpoints" + "golang.org/x/oauth2/google" +) + +type OAuth2Provider string + +const ( + OAuth2ProviderDefault OAuth2Provider = "" + OAuth2ProviderAzure OAuth2Provider = "azure" + OAuth2ProviderGoogle OAuth2Provider = "google" +) + +func (p OAuth2Provider) canonical() OAuth2Provider { + return OAuth2Provider(strings.ToLower(string(p))) +} + +// OAuth2 contains information about oauth2 authentication settings. +type OAuth2 struct { + // common oauth fields + ClientID string `config:"client_id"` + ClientSecret string `config:"client_secret"` + Enabled *bool `config:"enabled"` + EndpointParams map[string][]string `config:"endpoint_params"` + Provider OAuth2Provider `config:"provider"` + Scopes []string `config:"scopes"` + TokenURL string `config:"token_url"` + + // google specific + GoogleCredentialsFile string `config:"google_credentials_file"` + GoogleCredentialsJSON []byte `config:"google_credentials_json"` + GoogleJWTFile string `config:"google_jwt_file"` + + // microsoft azure specific + AzureTenantID string `config:"azure_tenant_id"` +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (o *OAuth2) IsEnabled() bool { + return o != nil && (o.Enabled == nil || *o.Enabled) +} + +func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, error) { + ctx = context.WithValue(ctx, oauth2.HTTPClient, client) + + if o.Provider != OAuth2ProviderGoogle { + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.GetTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.EndpointParams, + } + return creds.Client(ctx), nil + } + + creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) + } + + return oauth2.NewClient(ctx, creds.TokenSource), nil +} + +func (o *OAuth2) GetTokenURL() string { + if o.Provider == OAuth2ProviderAzure && o.TokenURL == "" { + return endpoints.AzureAD(o.AzureTenantID).TokenURL + } + return o.TokenURL +} + +func (o *OAuth2) validate() error { + switch o.Provider.canonical() { + case OAuth2ProviderAzure: + return o.validateAzureProvider() + case OAuth2ProviderGoogle: + return o.validateGoogleProvider() + case OAuth2ProviderDefault: + if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" { + return errors.New("invalid configuration: both token_url and client credentials must be provided") + } + default: + return fmt.Errorf("invalid configuration: unknown provider %q", o.Provider) + } + return nil +} + +func (o *OAuth2) validateGoogleProvider() error { + if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || + o.AzureTenantID != "" || len(o.EndpointParams) > 0 { + return errors.New("invalid configuration: none of token_url and client credentials can be used, use google_credentials_file or google_jwt_file instead") + } + + // credentials_json + if len(o.GoogleCredentialsJSON) > 0 { + return nil + } + + // credentials_file + if o.GoogleCredentialsFile != "" { + return o.populateCredentialsJSONFromFile(o.GoogleCredentialsFile) + } + + // jwt_file + if o.GoogleJWTFile != "" { + return o.populateCredentialsJSONFromFile(o.GoogleJWTFile) + } + + // Application Default Credentials (ADC) + ctx := context.Background() + if _, err := google.FindDefaultCredentials(ctx, o.Scopes...); err == nil { + return nil + } + + return fmt.Errorf("invalid configuration: no authentication credentials were configured or detected (ADC)") +} + +func (o *OAuth2) populateCredentialsJSONFromFile(file string) error { + if _, err := os.Stat(file); os.IsNotExist(err) { + return fmt.Errorf("the file %q cannot be found", file) + } + + credBytes, err := ioutil.ReadFile(file) + if err != nil { + return fmt.Errorf("the file %q cannot be read", file) + } + + o.GoogleCredentialsJSON = credBytes + + return nil +} + +func (o *OAuth2) validateAzureProvider() error { + if o.TokenURL == "" && o.AzureTenantID == "" { + return errors.New("invalid configuration: at least one of token_url or tenant_id must be provided") + } + if o.TokenURL != "" && o.AzureTenantID != "" { + return errors.New("invalid configuration: only one of token_url and tenant_id can be used") + } + if o.ClientID == "" || o.ClientSecret == "" { + return errors.New("invalid configuration: client credentials must be provided") + } + + return nil +} diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index ff4378f2beb..31829f5e233 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -17,8 +17,6 @@ import ( "time" "github.com/pkg/errors" - "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/input" @@ -469,20 +467,11 @@ func (in *HttpjsonInput) newHTTPClient(ctx context.Context) (*http.Client, error Timeout: in.config.HTTPClientTimeout, } - if in.config.OAuth2 == nil { - return client, nil + if in.config.OAuth2.IsEnabled() { + return in.config.OAuth2.Client(ctx, client) } - ctx = context.WithValue(ctx, oauth2.HTTPClient, client) - creds := clientcredentials.Config{ - ClientID: in.config.OAuth2.ClientID, - ClientSecret: in.config.OAuth2.ClientSecret, - TokenURL: in.config.OAuth2.TokenURL, - Scopes: in.config.OAuth2.Scopes, - EndpointParams: in.config.OAuth2.EndpointParams, - } - - return creds.Client(ctx), nil + return client, nil } func makeEvent(body string) beat.Event { From f25a6354a6acbd0ae72d02c01d6b34252ddb75de Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 5 Jun 2020 13:12:21 +0200 Subject: [PATCH 4/9] Change config as suggested and add config tests --- x-pack/filebeat/input/httpjson/config.go | 5 +- .../filebeat/input/httpjson/config_oauth.go | 66 +-- .../input/httpjson/config_oauth_test.go | 69 ++++ x-pack/filebeat/input/httpjson/config_test.go | 375 ++++++++++++++++++ .../filebeat/input/httpjson/httpjson_test.go | 94 ----- .../input/httpjson/testdata/credentials.json | 7 + 6 files changed, 490 insertions(+), 126 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/config_oauth_test.go create mode 100644 x-pack/filebeat/input/httpjson/config_test.go create mode 100644 x-pack/filebeat/input/httpjson/testdata/credentials.json diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index b3029ef488a..272f937a4cc 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -85,10 +85,7 @@ func (c *config) Validate() error { } if c.OAuth2 != nil { if c.APIKey != "" || c.AuthenticationScheme != "" { - return errors.Errorf("invalid configuration: both oauth2 and api_key or authentication_scheme cannot be set simultaneously") - } - if err := c.OAuth2.validate(); err != nil { - return err + return errors.Errorf("invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously") } } return nil diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index f9e1e3f3a8a..940ec45aa31 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -34,21 +34,21 @@ func (p OAuth2Provider) canonical() OAuth2Provider { // OAuth2 contains information about oauth2 authentication settings. type OAuth2 struct { // common oauth fields - ClientID string `config:"client_id"` - ClientSecret string `config:"client_secret"` + ClientID string `config:"client.id"` + ClientSecret string `config:"client.secret"` Enabled *bool `config:"enabled"` EndpointParams map[string][]string `config:"endpoint_params"` - Provider OAuth2Provider `config:"provider"` + Provider string `config:"provider"` Scopes []string `config:"scopes"` TokenURL string `config:"token_url"` // google specific - GoogleCredentialsFile string `config:"google_credentials_file"` - GoogleCredentialsJSON []byte `config:"google_credentials_json"` - GoogleJWTFile string `config:"google_jwt_file"` + GoogleCredentialsFile string `config:"google.credentials_file"` + GoogleCredentialsJSON []byte `config:"google.credentials_json"` + GoogleJWTFile string `config:"google.jwt_file"` // microsoft azure specific - AzureTenantID string `config:"azure_tenant_id"` + AzureTenantID string `config:"azure.tenant_id"` } // IsEnabled returns true if the `enable` field is set to true in the yaml. @@ -59,34 +59,43 @@ func (o *OAuth2) IsEnabled() bool { func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, error) { ctx = context.WithValue(ctx, oauth2.HTTPClient, client) - if o.Provider != OAuth2ProviderGoogle { - creds := clientcredentials.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - TokenURL: o.GetTokenURL(), - Scopes: o.Scopes, - EndpointParams: o.EndpointParams, + switch o.GetProvider() { + case OAuth2ProviderGoogle: + creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) } - return creds.Client(ctx), nil + return oauth2.NewClient(ctx, creds.TokenSource), nil } - creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) - if err != nil { - return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.GetTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.EndpointParams, } - return oauth2.NewClient(ctx, creds.TokenSource), nil + return creds.Client(ctx), nil } func (o *OAuth2) GetTokenURL() string { - if o.Provider == OAuth2ProviderAzure && o.TokenURL == "" { - return endpoints.AzureAD(o.AzureTenantID).TokenURL + switch o.GetProvider() { + case OAuth2ProviderAzure: + if o.TokenURL == "" { + return endpoints.AzureAD(o.AzureTenantID).TokenURL + } } + return o.TokenURL } -func (o *OAuth2) validate() error { - switch o.Provider.canonical() { +func (o OAuth2) GetProvider() OAuth2Provider { + return OAuth2Provider(o.Provider).canonical() +} + +func (o *OAuth2) Validate() error { + switch o.GetProvider() { case OAuth2ProviderAzure: return o.validateAzureProvider() case OAuth2ProviderGoogle: @@ -96,7 +105,7 @@ func (o *OAuth2) validate() error { return errors.New("invalid configuration: both token_url and client credentials must be provided") } default: - return fmt.Errorf("invalid configuration: unknown provider %q", o.Provider) + return fmt.Errorf("invalid configuration: unknown provider %q", o.GetProvider()) } return nil } @@ -104,7 +113,7 @@ func (o *OAuth2) validate() error { func (o *OAuth2) validateGoogleProvider() error { if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || o.AzureTenantID != "" || len(o.EndpointParams) > 0 { - return errors.New("invalid configuration: none of token_url and client credentials can be used, use google_credentials_file or google_jwt_file instead") + return errors.New("invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") } // credentials_json @@ -124,7 +133,8 @@ func (o *OAuth2) validateGoogleProvider() error { // Application Default Credentials (ADC) ctx := context.Background() - if _, err := google.FindDefaultCredentials(ctx, o.Scopes...); err == nil { + if creds, err := google.FindDefaultCredentials(ctx, o.Scopes...); err == nil { + o.GoogleCredentialsJSON = creds.JSON return nil } @@ -133,12 +143,12 @@ func (o *OAuth2) validateGoogleProvider() error { func (o *OAuth2) populateCredentialsJSONFromFile(file string) error { if _, err := os.Stat(file); os.IsNotExist(err) { - return fmt.Errorf("the file %q cannot be found", file) + return fmt.Errorf("invalid configuration: the file %q cannot be found", file) } credBytes, err := ioutil.ReadFile(file) if err != nil { - return fmt.Errorf("the file %q cannot be read", file) + return fmt.Errorf("invalid configuration: the file %q cannot be read", file) } o.GoogleCredentialsJSON = credBytes diff --git a/x-pack/filebeat/input/httpjson/config_oauth_test.go b/x-pack/filebeat/input/httpjson/config_oauth_test.go new file mode 100644 index 00000000000..3f818aedce3 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_oauth_test.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import "testing" + +func TestProviderCanonical(t *testing.T) { + const ( + a OAuth2Provider = "gOoGle" + b OAuth2Provider = "google" + ) + + if a.canonical() != b.canonical() { + t.Fatal("Canonical provider should be equal") + } +} + +func TestGetProviderIsCanonical(t *testing.T) { + const expected OAuth2Provider = "google" + + oauth2 := OAuth2{Provider: "GOogle"} + if oauth2.GetProvider() != expected { + t.Fatal("GetProvider should return canonical provider") + } +} + +func TestIsEnabled(t *testing.T) { + oauth2 := OAuth2{} + if !oauth2.IsEnabled() { + t.Fatal("OAuth2 should be enabled by default") + } + + var enabled = false + oauth2.Enabled = &enabled + + if oauth2.IsEnabled() { + t.Fatal("OAuth2 should be disabled") + } + + enabled = true + if !oauth2.IsEnabled() { + t.Fatal("OAuth2 should be enabled") + } +} + +func TestGetTokenURL(t *testing.T) { + const expected = "http://localhost" + oauth2 := OAuth2{TokenURL: "http://localhost"} + if oauth2.GetTokenURL() != expected { + t.Fatal("GetTokenURL should return the provided TokenURL") + } +} + +func TestGetTokenURLWithAzure(t *testing.T) { + const expectedWithoutTenantID = "http://localhost" + oauth2 := OAuth2{TokenURL: "http://localhost", Provider: "azure"} + if got := oauth2.GetTokenURL(); got != expectedWithoutTenantID { + t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) + } + + oauth2.TokenURL = "" + oauth2.AzureTenantID = "a_tenant_id" + const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" + if got := oauth2.GetTokenURL(); got != expectedWithTenantID { + t.Fatalf("GetTokenURL should return the generated TokenURL but got %q", got) + } +} diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go new file mode 100644 index 00000000000..1983a1f7cfb --- /dev/null +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -0,0 +1,375 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "os" + "testing" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestConfigValidationCase1(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "http_request_body": map[string]interface{}{"test": "abc"}, + "no_http_body": true, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. no_http_body and http_request_body cannot coexist.") + } +} + +func TestConfigValidationCase2(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "no_http_body": true, + "pagination": map[string]interface{}{"extra_body_content": map[string]interface{}{"test": "abc"}}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. no_http_body and pagination.extra_body_content cannot coexist.") + } +} + +func TestConfigValidationCase3(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "no_http_body": true, + "pagination": map[string]interface{}{"req_field": "abc"}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. no_http_body and pagination.req_field cannot coexist.") + } +} + +func TestConfigValidationCase4(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "req_field": "abc"}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. pagination.header and pagination.req_field cannot coexist.") + } +} + +func TestConfigValidationCase5(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "id_field": "abc"}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. pagination.header and pagination.id_field cannot coexist.") + } +} + +func TestConfigValidationCase6(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "extra_body_content": map[string]interface{}{"test": "abc"}}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. pagination.header and extra_body_content cannot coexist.") + } +} + +func TestConfigValidationCase7(t *testing.T) { + m := map[string]interface{}{ + "http_method": "DELETE", + "no_http_body": true, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") + } +} + +func TestConfigValidationCase8(t *testing.T) { + const expectedErr = "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config" + m := map[string]interface{}{ + "api_key": "an_api_key", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase9(t *testing.T) { + const expectedErr = "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config" + m := map[string]interface{}{ + "authentication_scheme": "an_api_key", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase10(t *testing.T) { + const expectedErr = "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{}, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase11(t *testing.T) { + const expectedErr = "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "unknown", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase12(t *testing.T) { + const expectedErr = "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase13(t *testing.T) { + const expectedErr = "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + "token_url": "localhost", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase14(t *testing.T) { + const expectedErr = "invalid configuration: client credentials must be provided accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase15(t *testing.T) { + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } +} + +func TestConfigValidationCase16(t *testing.T) { + const expectedErr = "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + "token_url": "localhost", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase17(t *testing.T) { + const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase18(t *testing.T) { + const expectedErr = "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./wrong", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase19(t *testing.T) { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") + const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase20(t *testing.T) { + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } +} + +func TestConfigValidationCase21(t *testing.T) { + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } +} + +func TestConfigValidationCase22(t *testing.T) { + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } +} + +func TestConfigValidationCase23(t *testing.T) { + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0", + }`), + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } +} diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 33643ac29ab..545cc4b40fa 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -187,100 +187,6 @@ func (o *stubOutleter) OnEvent(event beat.Event) bool { // --- Test Cases -func TestConfigValidationCase1(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "http_request_body": map[string]interface{}{"test": "abc"}, - "no_http_body": true, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and http_request_body cannot coexist.") - } -} - -func TestConfigValidationCase2(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "no_http_body": true, - "pagination": map[string]interface{}{"extra_body_content": map[string]interface{}{"test": "abc"}}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and pagination.extra_body_content cannot coexist.") - } -} - -func TestConfigValidationCase3(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "no_http_body": true, - "pagination": map[string]interface{}{"req_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. no_http_body and pagination.req_field cannot coexist.") - } -} - -func TestConfigValidationCase4(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "req_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and pagination.req_field cannot coexist.") - } -} - -func TestConfigValidationCase5(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "id_field": "abc"}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and pagination.id_field cannot coexist.") - } -} - -func TestConfigValidationCase6(t *testing.T) { - m := map[string]interface{}{ - "http_method": "GET", - "pagination": map[string]interface{}{"header": map[string]interface{}{"field_name": "Link", "regex_pattern": "<([^>]+)>; *rel=\"next\"(?:,|$)"}, "extra_body_content": map[string]interface{}{"test": "abc"}}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. pagination.header and extra_body_content cannot coexist.") - } -} - -func TestConfigValidationCase7(t *testing.T) { - m := map[string]interface{}{ - "http_method": "DELETE", - "no_http_body": true, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil { - t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") - } -} - func TestGetNextLinkFromHeader(t *testing.T) { header := make(http.Header) header.Add("Link", "; rel=\"self\"") diff --git a/x-pack/filebeat/input/httpjson/testdata/credentials.json b/x-pack/filebeat/input/httpjson/testdata/credentials.json new file mode 100644 index 00000000000..2b5fdd89e5c --- /dev/null +++ b/x-pack/filebeat/input/httpjson/testdata/credentials.json @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" +} From 9eb4f6269ea53a33becff4b2d4de0402df38146c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 8 Jun 2020 10:21:49 +0200 Subject: [PATCH 5/9] Add checks for invalid json in google validation --- .../filebeat/input/httpjson/config_oauth.go | 33 +++++++++++--- x-pack/filebeat/input/httpjson/config_test.go | 45 ++++++++++++++++++- .../testdata/invalid_credentials.json | 1 + 3 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index 940ec45aa31..8ce384234b4 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -6,6 +6,7 @@ package httpjson import ( "context" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -19,14 +20,20 @@ import ( "golang.org/x/oauth2/google" ) +// An OAuth2Provider represents a supported oauth provider. type OAuth2Provider string const ( - OAuth2ProviderDefault OAuth2Provider = "" - OAuth2ProviderAzure OAuth2Provider = "azure" - OAuth2ProviderGoogle OAuth2Provider = "google" + OAuth2ProviderDefault OAuth2Provider = "" // OAuth2ProviderDefault means no specific provider is set. + OAuth2ProviderAzure OAuth2Provider = "azure" // OAuth2ProviderAzure AzureAD. + OAuth2ProviderGoogle OAuth2Provider = "google" // OAuth2ProviderGoogle Google. ) +func (p *OAuth2Provider) Unpack(in string) error { + *p = OAuth2Provider(in) + return nil +} + func (p OAuth2Provider) canonical() OAuth2Provider { return OAuth2Provider(strings.ToLower(string(p))) } @@ -38,7 +45,7 @@ type OAuth2 struct { ClientSecret string `config:"client.secret"` Enabled *bool `config:"enabled"` EndpointParams map[string][]string `config:"endpoint_params"` - Provider string `config:"provider"` + Provider OAuth2Provider `config:"provider"` Scopes []string `config:"scopes"` TokenURL string `config:"token_url"` @@ -56,6 +63,7 @@ func (o *OAuth2) IsEnabled() bool { return o != nil && (o.Enabled == nil || *o.Enabled) } +// Client wraps the given http.Client and returns a new one that will use the oauth authentication. func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, error) { ctx = context.WithValue(ctx, oauth2.HTTPClient, client) @@ -79,6 +87,7 @@ func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, return creds.Client(ctx), nil } +// GetTokenURL returns the TokenURL. func (o *OAuth2) GetTokenURL() string { switch o.GetProvider() { case OAuth2ProviderAzure: @@ -90,10 +99,12 @@ func (o *OAuth2) GetTokenURL() string { return o.TokenURL } +// GetProvider returns provider in its canonical form. func (o OAuth2) GetProvider() OAuth2Provider { - return OAuth2Provider(o.Provider).canonical() + return o.Provider.canonical() } +// Validate checks if oauth2 config is valid. func (o *OAuth2) Validate() error { switch o.GetProvider() { case OAuth2ProviderAzure: @@ -110,6 +121,9 @@ func (o *OAuth2) Validate() error { return nil } +// findDefaultGoogleCredentials will default to google.FindDefaultCredentials and will only be changed for testing purposes +var findDefaultGoogleCredentials = google.FindDefaultCredentials + func (o *OAuth2) validateGoogleProvider() error { if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || o.AzureTenantID != "" || len(o.EndpointParams) > 0 { @@ -118,6 +132,9 @@ func (o *OAuth2) validateGoogleProvider() error { // credentials_json if len(o.GoogleCredentialsJSON) > 0 { + if !json.Valid(o.GoogleCredentialsJSON) { + return errors.New("invalid configuration: google.credentials_json must be valid JSON") + } return nil } @@ -133,7 +150,7 @@ func (o *OAuth2) validateGoogleProvider() error { // Application Default Credentials (ADC) ctx := context.Background() - if creds, err := google.FindDefaultCredentials(ctx, o.Scopes...); err == nil { + if creds, err := findDefaultGoogleCredentials(ctx, o.Scopes...); err == nil { o.GoogleCredentialsJSON = creds.JSON return nil } @@ -151,6 +168,10 @@ func (o *OAuth2) populateCredentialsJSONFromFile(file string) error { return fmt.Errorf("invalid configuration: the file %q cannot be read", file) } + if !json.Valid(credBytes) { + return fmt.Errorf("invalid configuration: the file %q does not contain valid JSON", file) + } + o.GoogleCredentialsJSON = credBytes return nil diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 1983a1f7cfb..5ad7b0a5e6a 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -5,9 +5,13 @@ package httpjson import ( + "context" "os" "testing" + "github.com/pkg/errors" + "golang.org/x/oauth2/google" + "github.com/elastic/beats/v7/libbeat/common" ) @@ -262,6 +266,13 @@ func TestConfigValidationCase16(t *testing.T) { } func TestConfigValidationCase17(t *testing.T) { + // we change the default function to force a failure + findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { + return nil, errors.New("failed") + } + + defer func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }() + const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" m := map[string]interface{}{ "oauth2": map[string]interface{}{ @@ -362,7 +373,7 @@ func TestConfigValidationCase23(t *testing.T) { "project_id": "foo", "private_key_id": "x", "client_email": "foo@bar.com", - "client_id": "0", + "client_id": "0" }`), }, "url": "localhost", @@ -373,3 +384,35 @@ func TestConfigValidationCase23(t *testing.T) { t.Fatalf("Configuration validation failed. no error expected but got %q", err) } } + +func TestConfigValidationCase24(t *testing.T) { + const expectedErr = "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`invalid`), + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} + +func TestConfigValidationCase25(t *testing.T) { + const expectedErr = "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'" + m := map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/invalid_credentials.json", + }, + "url": "localhost", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + } +} diff --git a/x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json b/x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json new file mode 100644 index 00000000000..9977a2836c1 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/testdata/invalid_credentials.json @@ -0,0 +1 @@ +invalid From 53e9b26b52f820ec7b030ddf6022b6294d742ffa Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 8 Jun 2020 11:51:31 +0200 Subject: [PATCH 6/9] Add documentation and azure.resource --- .../docs/inputs/input-httpjson.asciidoc | 142 ++++++++++++++++++ .../filebeat/input/httpjson/config_oauth.go | 20 ++- .../input/httpjson/config_oauth_test.go | 31 +++- 3 files changed, 188 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 61f1d249198..32751d0a5bc 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -48,6 +48,37 @@ Example configurations: url: http://localhost:9200/_search/scroll ---- +Additionally, it supports authentication via HTTP Headers, API key or oauth2. + +Example configurations with authentication: + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: httpjson + http_headers: + Authorization: 'Basic aGVsbG86d29ybGQ=' + url: http://localhost +---- + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: httpjson + api_key: 12345678901234567890abcdef + url: http://localhost +---- + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: httpjson + oauth2: + client.id: 12345678901234567890abcdef + client.secret: abcdef12345678901234567890 + token_url: http://localhost/oauth2/token + url: http://localhost +---- ==== Configuration options @@ -249,6 +280,117 @@ information. The URL of the HTTP API. Required. +[float] +==== `oauth2.enabled` + +The `enabled` setting can be used to disable the oauth2 configuration by +setting it to `false`. The default value is `true`. + +NOTE: OAuth2 settings are disabled if either `enabled` is set to `false` or +the `oauth2` section is missing. + +[float] +==== `oauth2.provider` + +The `provider` setting can be used to configure supported oauth2 providers. +Each supported provider will require specific settings. It is not set by default. + +NOTE: Supported providers are: `azure`, `google`. + +[float] +==== `oauth2.client.id` + +The `client.id` setting is used as part of the authentication flow. It is always required +except if using `google` as provider. + +NOTE: Required for providers: `default`, `azure`. + +[float] +==== `oauth2.client.secret` + +The `client.secret` setting is used as part of the authentication flow. It is always required +except if using `google` as provider. + +NOTE: Required for providers: `default`, `azure`. + +[float] +==== `oauth2.scopes` + +The `scopes` setting defines a list of scopes that will be requested during the oauth2 flow. +It is optional for all providers. + +[float] +==== `oauth2.token_url` + +The `token_url` setting specifies the endpoint that will be used to generate the +tokens during the oauth2 flow. It is required if no provider is specified. + +NOTE: For `azure` provider, a default `token_url` will be used if none provided, +but it can be set to override the default one. + +[float] +==== `oauth2.endpoint_params` + +The `endpoint_params` setting specifies a set of values that will be sent on each +request to the `token_url`. Each param key can have multiple values. + +NOTE: Can be set for all providers except `google`. + +["source","yaml",subs="attributes"] +---- +- type: httpjson + oauth2: + endpoint_params: + Param1: + - ValueA + - ValueB + Param2: + - Value +---- + +[float] +==== `oauth2.azure.tenant_id` + +The `azure.tenant_id` is used for authentication when using `azure` provider. +Since it is used in the process to generate the `token_url`, it can't be used in +combination with it. It is not required. + +NOTE: For information about where to find it, you can refer to +https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal + +[float] +==== `oauth2.azure.resource` + +The `azure.resource` is used to identify the accessed WebAPI resource when using `azure` provider. +It is not required. + +[float] +==== `oauth2.google.credentials_file` + +The `google.credentials_file` setting specifies the credentials file for Google. + +NOTE: Only one of the credentials settings can be set at once. If none is provided, loading +default credentials from the environment will be attempted via ADC. For for information about +how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. + +[float] +==== `oauth2.google.credentials_json` + +The `google.credentials_json` setting allows to write your credentials information as raw JSON. + +NOTE: Only one of the credentials settings can be set at once. If none is provided, loading +default credentials from the environment will be attempted via ADC. For for information about +how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. + +[float] +==== `oauth2.google.jwt_file` + +The `google.jwt_file` setting specifies the JWT Account Key file for Google. + +NOTE: Only one of the credentials settings can be set at once. If none is provided, loading +default credentials from the environment will be attempted via ADC. For for information about +how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. + [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index 8ce384234b4..a5bd0f378a4 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -56,6 +56,7 @@ type OAuth2 struct { // microsoft azure specific AzureTenantID string `config:"azure.tenant_id"` + AzureResource string `config:"azure.resource"` } // IsEnabled returns true if the `enable` field is set to true in the yaml. @@ -81,7 +82,7 @@ func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, ClientSecret: o.ClientSecret, TokenURL: o.GetTokenURL(), Scopes: o.Scopes, - EndpointParams: o.EndpointParams, + EndpointParams: o.GetEndpointParams(), } return creds.Client(ctx), nil @@ -104,6 +105,21 @@ func (o OAuth2) GetProvider() OAuth2Provider { return o.Provider.canonical() } +// GetEndpointParams returns endpoint params with any provider ones combined. +func (o OAuth2) GetEndpointParams() map[string][]string { + switch o.GetProvider() { + case OAuth2ProviderAzure: + if o.AzureResource != "" { + if o.EndpointParams == nil { + o.EndpointParams = map[string][]string{} + } + o.EndpointParams["resource"] = []string{o.AzureResource} + } + } + + return o.EndpointParams +} + // Validate checks if oauth2 config is valid. func (o *OAuth2) Validate() error { switch o.GetProvider() { @@ -126,7 +142,7 @@ var findDefaultGoogleCredentials = google.FindDefaultCredentials func (o *OAuth2) validateGoogleProvider() error { if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || - o.AzureTenantID != "" || len(o.EndpointParams) > 0 { + o.AzureTenantID != "" || o.AzureResource != "" || len(o.EndpointParams) > 0 { return errors.New("invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") } diff --git a/x-pack/filebeat/input/httpjson/config_oauth_test.go b/x-pack/filebeat/input/httpjson/config_oauth_test.go index 3f818aedce3..3fa0eed4284 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth_test.go +++ b/x-pack/filebeat/input/httpjson/config_oauth_test.go @@ -4,7 +4,10 @@ package httpjson -import "testing" +import ( + "reflect" + "testing" +) func TestProviderCanonical(t *testing.T) { const ( @@ -48,8 +51,8 @@ func TestIsEnabled(t *testing.T) { func TestGetTokenURL(t *testing.T) { const expected = "http://localhost" oauth2 := OAuth2{TokenURL: "http://localhost"} - if oauth2.GetTokenURL() != expected { - t.Fatal("GetTokenURL should return the provided TokenURL") + if got := oauth2.GetTokenURL(); got != expected { + t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) } } @@ -67,3 +70,25 @@ func TestGetTokenURLWithAzure(t *testing.T) { t.Fatalf("GetTokenURL should return the generated TokenURL but got %q", got) } } + +func TestGetEndpointParams(t *testing.T) { + var expected = map[string][]string{"foo": {"bar"}} + oauth2 := OAuth2{EndpointParams: map[string][]string{"foo": {"bar"}}} + if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expected) { + t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) + } +} + +func TestGetEndpointParamsWithAzure(t *testing.T) { + var expectedWithoutResource = map[string][]string{"foo": {"bar"}} + oauth2 := OAuth2{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} + if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expectedWithoutResource) { + t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) + } + + oauth2.AzureResource = "baz" + var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} + if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expectedWithResource) { + t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) + } +} From 7108a73882edb26e820bec9872d1276cdab9ced5 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 8 Jun 2020 14:39:04 +0200 Subject: [PATCH 7/9] Add oauth2 test and update changelog --- CHANGELOG.next.asciidoc | 1 + .../filebeat/input/httpjson/httpjson_test.go | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39cef21f4ce..ff8730d23c8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -44,6 +44,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359] - Adds check on `` config option value for the azure input `resource_manager_endpoint`. {pull}18890[18890] - Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953] +- Adds oauth support for httpjson input. {issue}18415[18415] {pull}18892[18892] *Heartbeat* diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 545cc4b40fa..3e3bf7f93ba 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -11,6 +11,7 @@ import ( "log" "net/http" "net/http/httptest" + "reflect" "regexp" "strconv" "sync" @@ -185,6 +186,50 @@ func (o *stubOutleter) OnEvent(event beat.Event) bool { return !o.done } +func newOAuth2TestServer(t *testing.T) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + + if r.Method != "POST" { + t.Errorf("expected POST request, got %v", r.Method) + return + } + + if err := r.ParseForm(); err != nil { + t.Errorf("no error expected, got %q", err) + return + } + + if gt := r.FormValue("grant_type"); gt != "client_credentials" { + t.Errorf("expected grant_type was client_credentials, got %q", gt) + return + } + + clientID := r.FormValue("client_id") + clientSecret := r.FormValue("client_secret") + if clientID == "" || clientSecret == "" { + clientID, clientSecret, _ = r.BasicAuth() + } + if clientID != "a_client_id" || clientSecret != "a_client_secret" { + t.Errorf("expected client credentials \"a_client_id:a_client_secret\", got \"%s:%s\"", clientID, clientSecret) + } + + if s := r.FormValue("scope"); s != "scope1 scope2" { + t.Errorf("expected scope was scope1+scope2, got %q", s) + return + } + + expectedParams := []string{"v1", "v2"} + if p := r.Form["param1"]; !reflect.DeepEqual(expectedParams, p) { + t.Errorf("expected params were %q, but got %q", expectedParams, p) + return + } + + w.Header().Set("content-type", "application/json") + w.Write([]byte(`{"token_type":"Bearer","expires_in":"3599","access_token":"abcdef1234567890"}`)) + })) +} + // --- Test Cases func TestGetNextLinkFromHeader(t *testing.T) { @@ -412,3 +457,34 @@ func TestRunStop(t *testing.T) { input.Stop() }) } + +func TestOAuth2(t *testing.T) { + ts := newOAuth2TestServer(t) + m := map[string]interface{}{ + "http_method": "GET", + "oauth2.client.id": "a_client_id", + "oauth2.client.secret": "a_client_secret", + "oauth2.token_url": ts.URL, + "oauth2.endpoint_params": map[string][]string{ + "param1": {"v1", "v2"}, + }, + "oauth2.scopes": []string{"scope1", "scope2"}, + "interval": 0, + } + defer ts.Close() + + runTest(t, false, false, m, func(input *HttpjsonInput, out *stubOutleter, t *testing.T) { + group, _ := errgroup.WithContext(context.Background()) + group.Go(input.run) + + events, ok := out.waitForEvents(1) + if !ok { + t.Fatalf("Expected 1 events, but got %d.", len(events)) + } + input.Stop() + + if err := group.Wait(); err != nil { + t.Fatal(err) + } + }) +} From 246f1d9c74fd571cdd53ff1328d85b7788df9cb2 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 9 Jun 2020 10:42:47 +0200 Subject: [PATCH 8/9] Address docs and change new test case into table tests --- .../docs/inputs/input-httpjson.asciidoc | 35 +- .../filebeat/input/httpjson/config_oauth.go | 21 +- x-pack/filebeat/input/httpjson/config_test.go | 516 ++++++++---------- 3 files changed, 254 insertions(+), 318 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 32751d0a5bc..8f47070b967 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -61,14 +61,6 @@ Example configurations with authentication: url: http://localhost ---- -["source","yaml",subs="attributes"] ----- -{beatname_lc}.inputs: -- type: httpjson - api_key: 12345678901234567890abcdef - url: http://localhost ----- - ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: @@ -294,24 +286,19 @@ the `oauth2` section is missing. The `provider` setting can be used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. - -NOTE: Supported providers are: `azure`, `google`. +Supported providers are: `azure`, `google`. [float] ==== `oauth2.client.id` The `client.id` setting is used as part of the authentication flow. It is always required -except if using `google` as provider. - -NOTE: Required for providers: `default`, `azure`. +except if using `google` as provider. Required for providers: `default`, `azure`. [float] ==== `oauth2.client.secret` The `client.secret` setting is used as part of the authentication flow. It is always required -except if using `google` as provider. - -NOTE: Required for providers: `default`, `azure`. +except if using `google` as provider. Required for providers: `default`, `azure`. [float] ==== `oauth2.scopes` @@ -325,16 +312,14 @@ It is optional for all providers. The `token_url` setting specifies the endpoint that will be used to generate the tokens during the oauth2 flow. It is required if no provider is specified. -NOTE: For `azure` provider, a default `token_url` will be used if none provided, -but it can be set to override the default one. +NOTE: For `azure` provider either `token_url` or `azure.tenant_id` is required. [float] ==== `oauth2.endpoint_params` The `endpoint_params` setting specifies a set of values that will be sent on each request to the `token_url`. Each param key can have multiple values. - -NOTE: Can be set for all providers except `google`. +Can be set for all providers except `google`. ["source","yaml",subs="attributes"] ---- @@ -355,8 +340,8 @@ The `azure.tenant_id` is used for authentication when using `azure` provider. Since it is used in the process to generate the `token_url`, it can't be used in combination with it. It is not required. -NOTE: For information about where to find it, you can refer to -https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal +For information about where to find it, you can refer to +https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal. [float] ==== `oauth2.azure.resource` @@ -370,7 +355,7 @@ It is not required. The `google.credentials_file` setting specifies the credentials file for Google. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading -default credentials from the environment will be attempted via ADC. For for information about +default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [float] @@ -379,7 +364,7 @@ how to provide Google credentials, please refer to https://cloud.google.com/docs The `google.credentials_json` setting allows to write your credentials information as raw JSON. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading -default credentials from the environment will be attempted via ADC. For for information about +default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [float] @@ -388,7 +373,7 @@ how to provide Google credentials, please refer to https://cloud.google.com/docs The `google.jwt_file` setting specifies the JWT Account Key file for Google. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading -default credentials from the environment will be attempted via ADC. For for information about +default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [id="{beatname_lc}-input-{type}-common-options"] diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index a5bd0f378a4..b9bdb45668c 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -69,23 +69,24 @@ func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, ctx = context.WithValue(ctx, oauth2.HTTPClient, client) switch o.GetProvider() { + case OAuth2ProviderAzure, OAuth2ProviderDefault: + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.GetTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.GetEndpointParams(), + } + return creds.Client(ctx), nil case OAuth2ProviderGoogle: creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) if err != nil { return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) } return oauth2.NewClient(ctx, creds.TokenSource), nil + default: + return nil, errors.New("oauth2 client: unknown provider") } - - creds := clientcredentials.Config{ - ClientID: o.ClientID, - ClientSecret: o.ClientSecret, - TokenURL: o.GetTokenURL(), - Scopes: o.Scopes, - EndpointParams: o.GetEndpointParams(), - } - - return creds.Client(ctx), nil } // GetTokenURL returns the TokenURL. diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 5ad7b0a5e6a..78432527f60 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -109,310 +109,260 @@ func TestConfigValidationCase7(t *testing.T) { } } -func TestConfigValidationCase8(t *testing.T) { - const expectedErr = "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config" - m := map[string]interface{}{ - "api_key": "an_api_key", - "oauth2": map[string]interface{}{ - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", +func TestConfigOauth2Validation(t *testing.T) { + cases := []struct { + name string + expectedErr string + input map[string]interface{} + setup func() + teardown func() + }{ + { + name: "can't set oauth2 and api_key together", + expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + input: map[string]interface{}{ + "api_key": "an_api_key", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase9(t *testing.T) { - const expectedErr = "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config" - m := map[string]interface{}{ - "authentication_scheme": "an_api_key", - "oauth2": map[string]interface{}{ - "token_url": "localhost", - "client": map[string]interface{}{ - "id": "a_client_id", - "secret": "a_client_secret", + { + name: "can't set oauth2 and authentication_scheme", + expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config", + input: map[string]interface{}{ + "authentication_scheme": "a_scheme", + "oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase10(t *testing.T) { - const expectedErr = "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{}, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase11(t *testing.T) { - const expectedErr = "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "unknown", + { + name: "token_url and client credentials must be set", + expectedErr: "invalid configuration: both token_url and client credentials must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{}, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase12(t *testing.T) { - const expectedErr = "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", + { + name: "must fail with an unknown provider", + expectedErr: "invalid configuration: unknown provider \"unknown\" accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "unknown", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase13(t *testing.T) { - const expectedErr = "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", - "azure.tenant_id": "a_tenant_id", - "token_url": "localhost", + { + name: "azure must have either tenant_id or token_url", + expectedErr: "invalid configuration: at least one of token_url or tenant_id must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase14(t *testing.T) { - const expectedErr = "invalid configuration: client credentials must be provided accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", - "azure.tenant_id": "a_tenant_id", + { + name: "azure must have only one of token_url and tenant_id", + expectedErr: "invalid configuration: only one of token_url and tenant_id can be used accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + "token_url": "localhost", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase15(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "azure", - "azure": map[string]interface{}{ - "tenant_id": "a_tenant_id", + { + name: "azure must have client credentials set", + expectedErr: "invalid configuration: client credentials must be provided accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + }, + "url": "localhost", }, - "client.id": "a_client_id", - "client.secret": "a_client_secret", }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase16(t *testing.T) { - const expectedErr = "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "azure": map[string]interface{}{ - "tenant_id": "a_tenant_id", + { + name: "azure config is valid", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "azure", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + }, + "url": "localhost", }, - "client.id": "a_client_id", - "client.secret": "a_client_secret", - "token_url": "localhost", }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase17(t *testing.T) { - // we change the default function to force a failure - findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { - return nil, errors.New("failed") - } - - defer func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }() - - const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", + { + name: "google can't have token_url or client credentials set", + expectedErr: "invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + "token_url": "localhost", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase18(t *testing.T) { - const expectedErr = "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./wrong", + { + name: "google must fail if no ADC available", + expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { + // we change the default function to force a failure + findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { + return nil, errors.New("failed") + } + }, + teardown: func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase19(t *testing.T) { - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") - const expectedErr = "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", + { + name: "google must fail if credentials file not found", + expectedErr: "invalid configuration: the file \"./wrong\" cannot be found accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./wrong", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase20(t *testing.T) { - os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", + { + name: "google must fail if ADC is wrongly set", + expectedErr: "invalid configuration: no authentication credentials were configured or detected (ADC) accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase21(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/credentials.json", + { + name: "google must work if ADC is set up", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + }, + "url": "localhost", + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase22(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.jwt_file": "./testdata/credentials.json", + { + name: "google must work if credentials_file is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase23(t *testing.T) { - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_json": []byte(`{ - "type": "service_account", - "project_id": "foo", - "private_key_id": "x", - "client_email": "foo@bar.com", - "client_id": "0" - }`), + { + name: "google must work if jwt_file is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } -} - -func TestConfigValidationCase24(t *testing.T) { - const expectedErr = "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_json": []byte(`invalid`), + { + name: "google must work if credentials_json is correct", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" + }`), + }, + "url": "localhost", + }, }, - "url": "localhost", - } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) - } -} - -func TestConfigValidationCase25(t *testing.T) { - const expectedErr = "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'" - m := map[string]interface{}{ - "oauth2": map[string]interface{}{ - "provider": "google", - "google.credentials_file": "./testdata/invalid_credentials.json", + { + name: "google must fail if credentials_json is not a valid JSON", + expectedErr: "invalid configuration: google.credentials_json must be valid JSON accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`invalid`), + }, + "url": "localhost", + }, + }, + { + name: "google must fail if the provided credentials file is not a valid JSON", + expectedErr: "invalid configuration: the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/invalid_credentials.json", + }, + "url": "localhost", + }, }, - "url": "localhost", } - cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() - if err := cfg.Unpack(&conf); err == nil || err.Error() != expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", expectedErr, err) + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + if c.setup != nil { + c.setup() + } + + if c.teardown != nil { + defer c.teardown() + } + + cfg := common.MustNewConfigFrom(c.input) + conf := defaultConfig() + err := cfg.Unpack(&conf) + + switch { + case c.expectedErr == "": + if err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } + + case c.expectedErr != "": + if err == nil || err.Error() != c.expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) + } + } + }) } } From dc1c1c44dd353d63320bcbb86fa06c751033b502 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 9 Jun 2020 10:56:07 +0200 Subject: [PATCH 9/9] Check if oauth2 is enabled in config.Validate and add test --- x-pack/filebeat/input/httpjson/config.go | 2 +- x-pack/filebeat/input/httpjson/config_test.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 272f937a4cc..bd9f584895b 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -83,7 +83,7 @@ func (c *config) Validate() error { } } } - if c.OAuth2 != nil { + if c.OAuth2.IsEnabled() { if c.APIKey != "" || c.AuthenticationScheme != "" { return errors.Errorf("invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously") } diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 78432527f60..cfec6a2440b 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -132,6 +132,21 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, + { + name: "can set oauth2 and api_key together if oauth2 is disabled", + input: map[string]interface{}{ + "api_key": "an_api_key", + "oauth2": map[string]interface{}{ + "enabled": false, + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + "url": "localhost", + }, + }, { name: "can't set oauth2 and authentication_scheme", expectedErr: "invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously accessing config",