From c4d55d383eaa87a85811256108fa205953526256 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 16 Oct 2020 10:08:28 +0200 Subject: [PATCH 01/35] Add oauth config --- x-pack/filebeat/input/httpjsonv2/auth.go | 262 +++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 x-pack/filebeat/input/httpjsonv2/auth.go diff --git a/x-pack/filebeat/input/httpjsonv2/auth.go b/x-pack/filebeat/input/httpjsonv2/auth.go new file mode 100644 index 00000000000..f88049d4eb9 --- /dev/null +++ b/x-pack/filebeat/input/httpjsonv2/auth.go @@ -0,0 +1,262 @@ +package httpjsonv2 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + "golang.org/x/oauth2/endpoints" + "golang.org/x/oauth2/google" +) + +type authConfig struct { + Basic *basicAuthConfig `config:"basic"` + OAuth2 *oAuth2Config `config:"oauth2"` +} + +func (c authConfig) Validate() error { + if c.Basic.isEnabled() && c.OAuth2.isEnabled() { + return errors.New("only one kind of auth can be enabled") + } + return nil +} + +type basicAuthConfig struct { + Enabled *bool `config:"enabled"` + User string `config:"user"` + Password string `config:"password"` +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (b *basicAuthConfig) isEnabled() bool { + return b != nil && (b.Enabled == nil || *b.Enabled) +} + +// Validate checks if oauth2 config is valid. +func (b *basicAuthConfig) Validate() error { + if !b.isEnabled() { + return nil + } + + if b.User == "" || b.Password == "" { + return errors.New("both user and password must be set") + } + + return nil +} + +// An oAuth2Provider represents a supported oauth provider. +type oAuth2Provider string + +const ( + oAuth2ProviderDefault oAuth2Provider = "" // oAuth2ProviderDefault means no specific provider is set. + oAuth2ProviderAzure oAuth2Provider = "azure" // oAuth2ProviderAzure AzureAD. + oAuth2ProviderGoogle oAuth2Provider = "google" // oAuth2ProviderGoogle Google. +) + +func (p *oAuth2Provider) Unpack(in string) error { + *p = oAuth2Provider(in) + return nil +} + +func (p oAuth2Provider) canonical() oAuth2Provider { + return oAuth2Provider(strings.ToLower(string(p))) +} + +type oAuth2Config struct { + Enabled *bool `config:"enabled"` + + // common oauth fields + ClientID string `config:"client.id"` + ClientSecret string `config:"client.secret"` + EndpointParams map[string][]string `config:"endpoint_params"` + Provider oAuth2Provider `config:"provider"` + Scopes []string `config:"scopes"` + TokenURL string `config:"token_url"` + + // google specific + GoogleCredentialsFile string `config:"google.credentials_file"` + GoogleCredentialsJSON []byte `config:"google.credentials_json"` + GoogleJWTFile string `config:"google.jwt_file"` + GoogleDelegatedAccount string `config:"google.delegated_account"` + + // microsoft azure specific + AzureTenantID string `config:"azure.tenant_id"` + AzureResource string `config:"azure.resource"` +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (o *oAuth2Config) isEnabled() bool { + return o != nil && (o.Enabled == nil || *o.Enabled) +} + +// Client wraps the given http.Client and returns a new one that will use the oauth authentication. +func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { + ctx = context.WithValue(ctx, oauth2.HTTPClient, client) + + switch o.GetProvider() { + case oAuth2ProviderAzure, oAuth2ProviderDefault: + creds := clientcredentials.Config{ + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + TokenURL: o.GetTokenURL(), + Scopes: o.Scopes, + EndpointParams: o.GetEndpointParams(), + } + return creds.Client(ctx), nil + case oAuth2ProviderGoogle: + if o.GoogleJWTFile != "" { + cfg, err := google.JWTConfigFromJSON(o.GoogleCredentialsJSON, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading jwt credentials: %w", err) + } + cfg.Subject = o.GoogleDelegatedAccount + return cfg.Client(ctx), nil + } + + creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) + } + return oauth2.NewClient(ctx, creds.TokenSource), nil + default: + return nil, errors.New("oauth2 client: unknown provider") + } +} + +// GetTokenURL returns the TokenURL. +func (o *oAuth2Config) GetTokenURL() string { + switch o.GetProvider() { + case oAuth2ProviderAzure: + if o.TokenURL == "" { + return endpoints.AzureAD(o.AzureTenantID).TokenURL + } + } + + return o.TokenURL +} + +// GetProvider returns provider in its canonical form. +func (o oAuth2Config) GetProvider() oAuth2Provider { + return o.Provider.canonical() +} + +// GetEndpointParams returns endpoint params with any provider ones combined. +func (o oAuth2Config) GetEndpointParams() map[string][]string { + switch o.GetProvider() { + case oAuth2ProviderAzure: + if o.AzureResource != "" { + if o.EndpointParams == nil { + o.EndpointParams = map[string][]string{} + } + o.EndpointParams["resource"] = []string{o.AzureResource} + } + } + + return o.EndpointParams +} + +// Validate checks if oauth2 config is valid. +func (o *oAuth2Config) Validate() error { + if !o.isEnabled() { + return nil + } + + switch o.GetProvider() { + case oAuth2ProviderAzure: + return o.validateAzureProvider() + case oAuth2ProviderGoogle: + return o.validateGoogleProvider() + case oAuth2ProviderDefault: + if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" { + return errors.New("both token_url and client credentials must be provided") + } + default: + return fmt.Errorf("unknown provider %q", o.GetProvider()) + } + + return nil +} + +// findDefaultGoogleCredentials will default to google.FindDefaultCredentials and will only be changed for testing purposes +var findDefaultGoogleCredentials = google.FindDefaultCredentials + +func (o *oAuth2Config) validateGoogleProvider() error { + if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || + o.AzureTenantID != "" || o.AzureResource != "" || len(o.EndpointParams) > 0 { + return errors.New("none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") + } + + // credentials_json + if len(o.GoogleCredentialsJSON) > 0 { + if o.GoogleDelegatedAccount != "" { + return errors.New("google.delegated_account can only be provided with a jwt_file") + } + if !json.Valid(o.GoogleCredentialsJSON) { + return errors.New("google.credentials_json must be valid JSON") + } + return nil + } + + // credentials_file + if o.GoogleCredentialsFile != "" { + if o.GoogleDelegatedAccount != "" { + return errors.New("google.delegated_account can only be provided with a jwt_file") + } + return o.populateCredentialsJSONFromFile(o.GoogleCredentialsFile) + } + + // jwt_file + if o.GoogleJWTFile != "" { + return o.populateCredentialsJSONFromFile(o.GoogleJWTFile) + } + + // Application Default Credentials (ADC) + ctx := context.Background() + if creds, err := findDefaultGoogleCredentials(ctx, o.Scopes...); err == nil { + o.GoogleCredentialsJSON = creds.JSON + return nil + } + + return fmt.Errorf("no authentication credentials were configured or detected (ADC)") +} + +func (o *oAuth2Config) populateCredentialsJSONFromFile(file string) error { + if _, err := os.Stat(file); os.IsNotExist(err) { + return fmt.Errorf("the file %q cannot be found", file) + } + + credBytes, err := ioutil.ReadFile(file) + if err != nil { + return fmt.Errorf("the file %q cannot be read", file) + } + + if !json.Valid(credBytes) { + return fmt.Errorf("the file %q does not contain valid JSON", file) + } + + o.GoogleCredentialsJSON = credBytes + + return nil +} + +func (o *oAuth2Config) validateAzureProvider() error { + if o.TokenURL == "" && o.AzureTenantID == "" { + return errors.New("at least one of token_url or tenant_id must be provided") + } + if o.TokenURL != "" && o.AzureTenantID != "" { + return errors.New("only one of token_url and tenant_id can be used") + } + if o.ClientID == "" || o.ClientSecret == "" { + return errors.New("client credentials must be provided") + } + + return nil +} From a19e762a6929058463db7178f1f1a1084acbd1f5 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 16 Oct 2020 10:08:55 +0200 Subject: [PATCH 02/35] Add stateless input --- .../input/httpjsonv2/input_manager.go | 37 ++++++++++++ .../input/httpjsonv2/input_stateless.go | 58 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 x-pack/filebeat/input/httpjsonv2/input_manager.go create mode 100644 x-pack/filebeat/input/httpjsonv2/input_stateless.go diff --git a/x-pack/filebeat/input/httpjsonv2/input_manager.go b/x-pack/filebeat/input/httpjsonv2/input_manager.go new file mode 100644 index 00000000000..542e9ed0064 --- /dev/null +++ b/x-pack/filebeat/input/httpjsonv2/input_manager.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjsonv2 + +import ( + "github.com/elastic/go-concert/unison" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/common" +) + +// inputManager wraps one stateless input manager +// and one cursor input manager. It will create one or the other +// based on the config that is passed. +type inputManager struct { + stateless *stateless.InputManager +} + +var _ v2.InputManager = inputManager{} + +// Init initializes both wrapped input managers. +func (m inputManager) Init(grp unison.Group, mode v2.Mode) error { + return m.stateless.Init(grp, mode) // multierr.Append() +} + +// Create creates a cursor input manager if the config has a date cursor set up, +// otherwise it creates a stateless input manager. +func (m inputManager) Create(cfg *common.Config) (v2.Input, error) { + var config config + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + return m.stateless.Create(cfg) +} diff --git a/x-pack/filebeat/input/httpjsonv2/input_stateless.go b/x-pack/filebeat/input/httpjsonv2/input_stateless.go new file mode 100644 index 00000000000..3788e072965 --- /dev/null +++ b/x-pack/filebeat/input/httpjsonv2/input_stateless.go @@ -0,0 +1,58 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjsonv2 + +import ( + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" +) + +type statelessInput struct { + config config + tlsConfig *tlscommon.TLSConfig +} + +func (statelessInput) Name() string { + return "httpjson-stateless" +} + +func statelessConfigure(cfg *common.Config) (stateless.Input, error) { + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + return nil, err + } + return newStatelessInput(conf) +} + +func newStatelessInput(config config) (*statelessInput, error) { + tlsConfig, err := newTLSConfig(config) + if err != nil { + return nil, err + } + return &statelessInput{config: config, tlsConfig: tlsConfig}, nil +} + +func (in *statelessInput) Test(v2.TestContext) error { + return test(in.config.Request.URL.URL) +} + +type statelessPublisher struct { + wrapped stateless.Publisher +} + +func (pub statelessPublisher) Publish(event beat.Event, _ interface{}) error { + pub.wrapped.Publish(event) + return nil +} + +// Run starts the input and blocks until it ends the execution. +// It will return on context cancellation, any other error will be retried. +func (in *statelessInput) Run(ctx v2.Context, publisher stateless.Publisher) error { + pub := statelessPublisher{wrapped: publisher} + return run(ctx, in.config, in.tlsConfig, pub, nil) +} From 19b8941a19f95c3a427220ee02bae344f702d4ea Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 16 Oct 2020 10:10:45 +0200 Subject: [PATCH 03/35] Add request and basic config --- x-pack/filebeat/input/httpjsonv2/config.go | 30 ++++++ .../httpjsonv2/{auth.go => config_auth.go} | 0 .../input/httpjsonv2/config_request.go | 94 +++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 x-pack/filebeat/input/httpjsonv2/config.go rename x-pack/filebeat/input/httpjsonv2/{auth.go => config_auth.go} (100%) create mode 100644 x-pack/filebeat/input/httpjsonv2/config_request.go diff --git a/x-pack/filebeat/input/httpjsonv2/config.go b/x-pack/filebeat/input/httpjsonv2/config.go new file mode 100644 index 00000000000..1c3d806bd68 --- /dev/null +++ b/x-pack/filebeat/input/httpjsonv2/config.go @@ -0,0 +1,30 @@ +package httpjsonv2 + +import ( + "errors" + "time" +) + +type config struct { + Interval time.Duration `config:"interval" validate:"required"` + Auth *authConfig `config:"auth"` + Request *requestConfig `config:"request" validate:"required"` + // Response *responseConfig `config:"response"` +} + +func (c config) Validate() error { + if c.Interval <= 0 { + return errors.New("interval must be greater than 0") + } + return nil +} + +func defaultConfig() config { + return config{ + Interval: time.Minute, + Auth: &authConfig{}, + Request: &requestConfig{ + Method: "GET", + }, + } +} diff --git a/x-pack/filebeat/input/httpjsonv2/auth.go b/x-pack/filebeat/input/httpjsonv2/config_auth.go similarity index 100% rename from x-pack/filebeat/input/httpjsonv2/auth.go rename to x-pack/filebeat/input/httpjsonv2/config_auth.go diff --git a/x-pack/filebeat/input/httpjsonv2/config_request.go b/x-pack/filebeat/input/httpjsonv2/config_request.go new file mode 100644 index 00000000000..b195f4cfe82 --- /dev/null +++ b/x-pack/filebeat/input/httpjsonv2/config_request.go @@ -0,0 +1,94 @@ +package httpjsonv2 + +import ( + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjsonv2/internal/transforms" +) + +type retryConfig struct { + MaxAttempts *int `config:"max_attempts"` + WaitMin *time.Duration `config:"wait_min"` + WaitMax *time.Duration `config:"wait_max"` +} + +func (c retryConfig) Validate() error { + switch { + case c.MaxAttempts != nil && *c.MaxAttempts <= 0: + return errors.New("max_attempts must be greater than 0") + case c.WaitMin != nil && *c.WaitMin <= 0: + return errors.New("wait_min must be greater than 0") + case c.WaitMax != nil && *c.WaitMax <= 0: + return errors.New("wait_max must be greater than 0") + } + return nil +} + +type rateLimitConfig struct { + Limit *transforms.Template `config:"limit"` + Reset *transforms.Template `config:"reset"` + Remaining *transforms.Template `config:"remaining"` +} + +func (c rateLimitConfig) Validate() error { + if c.Limit == nil || c.Reset == nil || c.Remaining == nil { + return errors.New("all rate_limit fields must have a value") + } + + return nil +} + +type urlConfig struct { + *url.URL +} + +func (u *urlConfig) Unpack(in string) error { + parsed, err := url.Parse(in) + if err != nil { + return err + } + + *u = urlConfig{URL: parsed} + + return nil +} + +type requestConfig struct { + URL *urlConfig `config:"url" validate:"required"` + Method string `config:"method" validate:"required"` + Body *common.MapStr `config:"body"` + Timeout *time.Duration `config:"timeout"` + SSL *tlscommon.Config `config:"ssl"` + Retry retryConfig `config:"retry"` + RateLimit *rateLimitConfig `config:"rate_limit"` + Transforms transforms.Config `config:"transforms"` +} + +func (c requestConfig) Validate() error { + + switch strings.ToUpper(c.Method) { + case "POST": + case "GET": + if c.Body != nil { + return errors.New("body can't be used with method: \"GET\"") + } + default: + return fmt.Errorf("unsupported method %q", c.Method) + } + + if c.Timeout != nil && *c.Timeout <= 0 { + return errors.New("timeout must be greater than 0") + } + + if _, err := transforms.New(c.Transforms, "request"); err != nil { + return err + } + + return nil +} From cfba7744e5952c6c997c8e44b61ed049e2a119fa Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 20 Oct 2020 15:24:43 +0200 Subject: [PATCH 04/35] Create v2 input and basic transforms --- .../{httpjsonv2 => httpjson/v2}/config.go | 10 +- .../v2}/config_auth.go | 2 +- .../v2}/config_request.go | 12 +- .../input/httpjson/v2/config_response.go | 17 ++ x-pack/filebeat/input/httpjson/v2/input.go | 215 ++++++++++++++++++ .../v2}/input_manager.go | 2 +- .../v2}/input_stateless.go | 2 +- .../v2/internal/transforms/append/append.go | 92 ++++++++ .../v2/internal/transforms/delete/delete.go | 72 ++++++ .../v2/internal/transforms/registry.go | 77 +++++++ .../v2/internal/transforms/set/set.go | 92 ++++++++ .../v2/internal/transforms/template.go | 53 +++++ .../v2/internal/transforms/template_funcs.go | 66 ++++++ .../v2/internal/transforms/transform.go | 182 +++++++++++++++ x-pack/filebeat/input/httpjson/v2/request.go | 155 +++++++++++++ x-pack/filebeat/input/httpjson/v2/response.go | 104 +++++++++ 16 files changed, 1139 insertions(+), 14 deletions(-) rename x-pack/filebeat/input/{httpjsonv2 => httpjson/v2}/config.go (58%) rename x-pack/filebeat/input/{httpjsonv2 => httpjson/v2}/config_auth.go (99%) rename x-pack/filebeat/input/{httpjsonv2 => httpjson/v2}/config_request.go (88%) create mode 100644 x-pack/filebeat/input/httpjson/v2/config_response.go create mode 100644 x-pack/filebeat/input/httpjson/v2/input.go rename x-pack/filebeat/input/{httpjsonv2 => httpjson/v2}/input_manager.go (98%) rename x-pack/filebeat/input/{httpjsonv2 => httpjson/v2}/input_stateless.go (98%) create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go create mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go create mode 100644 x-pack/filebeat/input/httpjson/v2/request.go create mode 100644 x-pack/filebeat/input/httpjson/v2/response.go diff --git a/x-pack/filebeat/input/httpjsonv2/config.go b/x-pack/filebeat/input/httpjson/v2/config.go similarity index 58% rename from x-pack/filebeat/input/httpjsonv2/config.go rename to x-pack/filebeat/input/httpjson/v2/config.go index 1c3d806bd68..71ee0e96538 100644 --- a/x-pack/filebeat/input/httpjsonv2/config.go +++ b/x-pack/filebeat/input/httpjson/v2/config.go @@ -1,4 +1,4 @@ -package httpjsonv2 +package v2 import ( "errors" @@ -6,10 +6,10 @@ import ( ) type config struct { - Interval time.Duration `config:"interval" validate:"required"` - Auth *authConfig `config:"auth"` - Request *requestConfig `config:"request" validate:"required"` - // Response *responseConfig `config:"response"` + Interval time.Duration `config:"interval" validate:"required"` + Auth *authConfig `config:"auth"` + Request *requestConfig `config:"request" validate:"required"` + Response *responseConfig `config:"response"` } func (c config) Validate() error { diff --git a/x-pack/filebeat/input/httpjsonv2/config_auth.go b/x-pack/filebeat/input/httpjson/v2/config_auth.go similarity index 99% rename from x-pack/filebeat/input/httpjsonv2/config_auth.go rename to x-pack/filebeat/input/httpjson/v2/config_auth.go index f88049d4eb9..5dd90c122f7 100644 --- a/x-pack/filebeat/input/httpjsonv2/config_auth.go +++ b/x-pack/filebeat/input/httpjson/v2/config_auth.go @@ -1,4 +1,4 @@ -package httpjsonv2 +package v2 import ( "context" diff --git a/x-pack/filebeat/input/httpjsonv2/config_request.go b/x-pack/filebeat/input/httpjson/v2/config_request.go similarity index 88% rename from x-pack/filebeat/input/httpjsonv2/config_request.go rename to x-pack/filebeat/input/httpjson/v2/config_request.go index b195f4cfe82..a0a96b67906 100644 --- a/x-pack/filebeat/input/httpjsonv2/config_request.go +++ b/x-pack/filebeat/input/httpjson/v2/config_request.go @@ -1,4 +1,4 @@ -package httpjsonv2 +package v2 import ( "errors" @@ -9,7 +9,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjsonv2/internal/transforms" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" ) type retryConfig struct { @@ -70,9 +70,9 @@ type requestConfig struct { Transforms transforms.Config `config:"transforms"` } -func (c requestConfig) Validate() error { - - switch strings.ToUpper(c.Method) { +func (c *requestConfig) Validate() error { + c.Method = strings.ToUpper(c.Method) + switch c.Method { case "POST": case "GET": if c.Body != nil { @@ -86,7 +86,7 @@ func (c requestConfig) Validate() error { return errors.New("timeout must be greater than 0") } - if _, err := transforms.New(c.Transforms, "request"); err != nil { + if _, err := transforms.New(c.Transforms, requestNamespace); err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/v2/config_response.go b/x-pack/filebeat/input/httpjson/v2/config_response.go new file mode 100644 index 00000000000..e174c2addc8 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/config_response.go @@ -0,0 +1,17 @@ +package v2 + +import ( + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" +) + +type responseConfig struct { + Transforms transforms.Config `config:"transforms"` +} + +func (c *responseConfig) Validate() error { + if _, err := transforms.New(c.Transforms, responseNamespace); err != nil { + return err + } + + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go new file mode 100644 index 00000000000..236a01b9711 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -0,0 +1,215 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "reflect" + "time" + + "github.com/hashicorp/go-retryablehttp" + "go.uber.org/zap" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/v7/libbeat/common/useragent" + "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/timed" +) + +const ( + inputName = "httpjson" +) + +var ( + userAgent = useragent.UserAgent("Filebeat") + + // for testing + timeNow = time.Now +) + +type retryLogger struct { + log *logp.Logger +} + +func newRetryLogger() *retryLogger { + return &retryLogger{ + log: logp.NewLogger("httpjson.retryablehttp", zap.AddCallerSkip(1)), + } +} + +func (log *retryLogger) Error(format string, args ...interface{}) { + log.log.Errorf(format, args...) +} + +func (log *retryLogger) Info(format string, args ...interface{}) { + log.log.Infof(format, args...) +} + +func (log *retryLogger) Debug(format string, args ...interface{}) { + log.log.Debugf(format, args...) +} + +func (log *retryLogger) Warn(format string, args ...interface{}) { + log.log.Warnf(format, args...) +} + +func Plugin(log *logp.Logger, store cursor.StateStore) v2.Plugin { + sim := stateless.NewInputManager(statelessConfigure) + + registerRequestTransforms() + registerResponseTransforms() + + return v2.Plugin{ + Name: inputName, + Stability: feature.Beta, + Deprecated: false, + Manager: inputManager{ + stateless: &sim, + }, + } +} + +func newTLSConfig(config config) (*tlscommon.TLSConfig, error) { + if err := config.Validate(); err != nil { + return nil, err + } + + tlsConfig, err := tlscommon.LoadTLSConfig(config.Request.SSL) + if err != nil { + return nil, err + } + + return tlsConfig, nil +} + +func test(url *url.URL) error { + port := func() string { + if url.Port() != "" { + return url.Port() + } + switch url.Scheme { + case "https": + return "443" + } + return "80" + }() + + _, err := net.DialTimeout("tcp", net.JoinHostPort(url.Hostname(), port), time.Second) + if err != nil { + return fmt.Errorf("url %q is unreachable", url) + } + + return nil +} + +func run( + ctx v2.Context, + config config, + tlsConfig *tlscommon.TLSConfig, + publisher cursor.Publisher, + cursor *cursor.Cursor, +) error { + log := ctx.Logger.With("url", config.Request.URL) + + stdCtx := ctxtool.FromCanceller(ctx.Cancelation) + + httpClient, err := newHTTPClient(stdCtx, config, tlsConfig) + if err != nil { + return err + } + + requestFactory := newRequestFactory(config.Request, config.Auth, log) + responseProcessor := newResponseProcessor(config.Response) + + requester := newRequester(httpClient, requestFactory, responseProcessor, log) + + err = timed.Periodic(stdCtx, config.Interval, func() error { + log.Info("Process another repeated request.") + + err := requester.processRequest(stdCtx, publisher) + if err == nil { + return nil + } + + if stdCtx.Err() != nil { + return err + } + + log.Errorf("Error while processing http request: %v", err) + + return nil + }) + + log.Infof("Context done: %v", err) + + return nil +} + +func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig) (*http.Client, error) { + timeout := getValFromPtr(config.Request.Timeout, time.Duration(0)).(time.Duration) + + // Make retryable HTTP client + client := &retryablehttp.Client{ + HTTPClient: &http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: timeout, + }).DialContext, + TLSClientConfig: tlsConfig.ToConfig(), + DisableKeepAlives: true, + }, + Timeout: timeout, + }, + Logger: newRetryLogger(), + RetryWaitMin: getValFromPtr(config.Request.Retry.WaitMin, time.Duration(0)).(time.Duration), + RetryWaitMax: getValFromPtr(config.Request.Retry.WaitMax, time.Duration(0)).(time.Duration), + RetryMax: getValFromPtr(config.Request.Retry.MaxAttempts, 0).(int), + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + } + + if config.Auth.OAuth2.isEnabled() { + return config.Auth.OAuth2.client(ctx, client.StandardClient()) + } + + return client.StandardClient(), nil +} + +func getValFromPtr(ptr, defaultVal interface{}) interface{} { + switch { + case reflect.ValueOf(ptr).Kind() != reflect.Ptr, + !reflect.ValueOf(ptr).IsValid(), + reflect.ValueOf(ptr).IsNil(): + return defaultVal + } + + return reflect.Indirect(reflect.ValueOf(ptr)) +} + +func makeEvent(body string) beat.Event { + now := timeNow() + fields := common.MapStr{ + "event": common.MapStr{ + "created": now, + }, + "message": body, + } + + return beat.Event{ + Timestamp: now, + Fields: fields, + } +} diff --git a/x-pack/filebeat/input/httpjsonv2/input_manager.go b/x-pack/filebeat/input/httpjson/v2/input_manager.go similarity index 98% rename from x-pack/filebeat/input/httpjsonv2/input_manager.go rename to x-pack/filebeat/input/httpjson/v2/input_manager.go index 542e9ed0064..971740ccab8 100644 --- a/x-pack/filebeat/input/httpjsonv2/input_manager.go +++ b/x-pack/filebeat/input/httpjson/v2/input_manager.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package httpjsonv2 +package v2 import ( "github.com/elastic/go-concert/unison" diff --git a/x-pack/filebeat/input/httpjsonv2/input_stateless.go b/x-pack/filebeat/input/httpjson/v2/input_stateless.go similarity index 98% rename from x-pack/filebeat/input/httpjsonv2/input_stateless.go rename to x-pack/filebeat/input/httpjson/v2/input_stateless.go index 3788e072965..92a1b8ae2dd 100644 --- a/x-pack/filebeat/input/httpjsonv2/input_stateless.go +++ b/x-pack/filebeat/input/httpjson/v2/input_stateless.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package httpjsonv2 +package v2 import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go new file mode 100644 index 00000000000..1a64e01a649 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go @@ -0,0 +1,92 @@ +package append + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" + "github.com/pkg/errors" +) + +const Name = "append" + +type config struct { + Target string `config:"target"` + Value *transforms.Template `config:"value"` + Default string `config:"default"` +} + +type appendTransform struct { + targetInfo transforms.TargetInfo + value *transforms.Template + defaultValue string + + run func(tr *transforms.Transformable, key, val string) error +} + +func New(cfg *common.Config) (transforms.Transform, error) { + c := &config{} + if err := cfg.Unpack(c); err != nil { + return nil, errors.Wrap(err, "fail to unpack the append configuration") + } + app := &appendTransform{ + targetInfo: transforms.GetTargetInfo(c.Target), + value: c.Value, + defaultValue: c.Default, + } + + switch app.targetInfo.Type { + // case transforms.TargetCursor: + case transforms.TargetBody: + app.run = runBody + case transforms.TargetHeaders: + app.run = runHeader + case transforms.TargetURLParams: + app.run = runURLParams + case transforms.TargetURLValue: + return nil, errors.New("can't append to url.value") + default: + return nil, errors.New("unknown target type") + } + + return app, nil +} + +func (appendTransform) String() string { return Name } + +func (app *appendTransform) Run(tr *transforms.Transformable) (*transforms.Transformable, error) { + value := app.value.Execute(tr, app.defaultValue) + return tr, app.run(tr, app.targetInfo.Name, value) +} + +func appendToCommonMap(m common.MapStr, key, val string) error { + var value interface{} = val + if found, _ := m.HasKey(key); found { + prev, _ := m.GetValue(key) + switch t := prev.(type) { + case []string: + value = append(t, val) + case []interface{}: + value = append(t, val) + default: + value = []interface{}{prev, val} + } + + } + if _, err := m.Put(key, value); err != nil { + return err + } + return nil +} + +func runBody(tr *transforms.Transformable, key, value string) error { + return appendToCommonMap(tr.Body, key, value) +} + +func runHeader(tr *transforms.Transformable, key, value string) error { + tr.Headers.Add(key, value) + return nil +} + +func runURLParams(tr *transforms.Transformable, key, value string) error { + tr.URL.Query().Add(key, value) + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go new file mode 100644 index 00000000000..6b41cb0420b --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go @@ -0,0 +1,72 @@ +package delete + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" + "github.com/pkg/errors" +) + +const Name = "delete" + +type config struct { + Target string `config:"target"` +} + +type delete struct { + targetInfo transforms.TargetInfo + + run func(tr *transforms.Transformable, key string) error +} + +func New(cfg *common.Config) (transforms.Transform, error) { + c := &config{} + if err := cfg.Unpack(c); err != nil { + return nil, errors.Wrap(err, "fail to unpack the set configuration") + } + delete := &delete{ + targetInfo: transforms.GetTargetInfo(c.Target), + } + + switch delete.targetInfo.Type { + // case transforms.TargetCursor: + case transforms.TargetBody: + delete.run = runBody + case transforms.TargetHeaders: + delete.run = runHeader + case transforms.TargetURLParams: + delete.run = runURLParams + case transforms.TargetURLValue: + return nil, errors.New("can't append to url.value") + default: + return nil, errors.New("unknown target type") + } + + return delete, nil +} + +func (delete) String() string { return Name } + +func (delete *delete) Run(tr *transforms.Transformable) (*transforms.Transformable, error) { + return tr, delete.run(tr, delete.targetInfo.Name) +} + +func deleteFromCommonMap(m common.MapStr, key string) error { + if err := m.Delete(key); err != common.ErrKeyNotFound { + return err + } + return nil +} + +func runBody(tr *transforms.Transformable, key string) error { + return deleteFromCommonMap(tr.Body, key) +} + +func runHeader(tr *transforms.Transformable, key string) error { + tr.Headers.Del(key) + return nil +} + +func runURLParams(tr *transforms.Transformable, key string) error { + tr.URL.Query().Del(key) + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go new file mode 100644 index 00000000000..77ffa8203cc --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go @@ -0,0 +1,77 @@ +package transforms + +import ( + "errors" + "fmt" + "strings" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type Constructor func(config *common.Config) (Transform, error) + +var registeredTransforms = newRegistry() + +type registry struct { + namespaces map[string]map[string]Constructor +} + +func newRegistry() *registry { + return ®istry{namespaces: make(map[string]map[string]Constructor)} +} + +func (reg *registry) register(namespace, transform string, constructor Constructor) error { + if constructor == nil { + return errors.New("constructor can't be nil") + } + + m, found := reg.namespaces[namespace] + if !found { + reg.namespaces[namespace] = make(map[string]Constructor) + m = reg.namespaces[namespace] + } + + if _, found := m[transform]; found { + return errors.New("already registered") + } + + m[transform] = constructor + + return nil +} + +func (reg registry) String() string { + if len(reg.namespaces) == 0 { + return "(empty registry)" + } + + var str string + for namespace, m := range reg.namespaces { + var names []string + for k := range m { + names = append(names, k) + } + str += fmt.Sprintf("%s: (%s)\n", namespace, strings.Join(names, ", ")) + } + + return str +} + +func (reg registry) get(namespace, transform string) (Constructor, bool) { + m, found := reg.namespaces[namespace] + if !found { + return nil, false + } + c, found := m[transform] + return c, found +} + +func RegisterTransform(namespace, transform string, constructor Constructor) { + logp.L().Named(logName).Debugf("Register transform %s:%s", namespace, transform) + + err := registeredTransforms.register(namespace, transform, constructor) + if err != nil { + panic(err) + } +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go new file mode 100644 index 00000000000..3c23ab88758 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go @@ -0,0 +1,92 @@ +package set + +import ( + "net/url" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" + "github.com/pkg/errors" +) + +const Name = "set" + +type config struct { + Target string `config:"target"` + Value *transforms.Template `config:"value"` + Default string `config:"default"` +} + +type set struct { + targetInfo transforms.TargetInfo + value *transforms.Template + defaultValue string + + run func(tr *transforms.Transformable, key, val string) error +} + +func New(cfg *common.Config) (transforms.Transform, error) { + c := &config{} + if err := cfg.Unpack(c); err != nil { + return nil, errors.Wrap(err, "fail to unpack the set configuration") + } + set := &set{ + targetInfo: transforms.GetTargetInfo(c.Target), + value: c.Value, + defaultValue: c.Default, + } + + switch set.targetInfo.Type { + // case transforms.TargetCursor: + case transforms.TargetBody: + set.run = runBody + case transforms.TargetHeaders: + set.run = runHeader + case transforms.TargetURLValue: + set.run = runURLValue + case transforms.TargetURLParams: + set.run = runURLParams + default: + return nil, errors.New("unknown target type") + } + + return set, nil +} + +func (set) String() string { return Name } + +func (set *set) Run(tr *transforms.Transformable) (*transforms.Transformable, error) { + value := set.value.Execute(tr, set.defaultValue) + return tr, set.run(tr, set.targetInfo.Name, value) +} + +func setToCommonMap(m common.MapStr, key, val string) error { + if _, err := m.Put(key, val); err != nil { + return err + } + return nil +} + +func runBody(tr *transforms.Transformable, key, value string) error { + return setToCommonMap(tr.Body, key, value) +} + +func runHeader(tr *transforms.Transformable, key, value string) error { + tr.Headers.Add(key, value) + return nil +} + +func runURLParams(tr *transforms.Transformable, key, value string) error { + tr.URL.Query().Add(key, value) + return nil +} + +func runURLValue(tr *transforms.Transformable, _, value string) error { + query := tr.URL.Query().Encode() + url, err := url.Parse(value) + if err != nil { + return err + } + url.RawQuery = query + tr.URL = *url + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go new file mode 100644 index 00000000000..ab664af17d2 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go @@ -0,0 +1,53 @@ +package transforms + +import ( + "bytes" + "html/template" +) + +type Template struct { + *template.Template +} + +func (t *Template) Unpack(in string) error { + tpl, err := template.New(""). + Option("missingkey=error"). + Funcs(template.FuncMap{ + "now": now, + "formatDate": formatDate, + "parseDate": parseDate, + "getRFC5988Link": getRFC5988Link, + }). + Parse(in) + if err != nil { + return err + } + + *t = Template{Template: tpl} + + return nil +} + +func (t *Template) Execute(tr *Transformable, defaultVal string) (val string) { + defer func() { + if r := recover(); r != nil { + // really ugly + val = defaultVal + } + }() + + buf := new(bytes.Buffer) + data := map[string]interface{}{ + "header": tr.Headers.Clone(), + "body": tr.Body.Clone(), + "url.value": tr.URL.String(), + "url.params": tr.URL.Query(), + // "cursor": tr.Cursor.Clone(), + // "last_event": tr.LastEvent, + // "last_response": tr.LastResponse.Clone(), + } + if err := t.Template.Execute(buf, data); err != nil { + return defaultVal + } + return buf.String() +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go new file mode 100644 index 00000000000..552b71bda04 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go @@ -0,0 +1,66 @@ +package transforms + +import "time" + +var ( + predefinedLayouts = map[string]string{ + "ANSIC": time.ANSIC, + "UnixDate": time.UnixDate, + "RubyDate": time.RubyDate, + "RFC822": time.RFC822, + "RFC822Z": time.RFC822Z, + "RFC850": time.RFC850, + "RFC1123": time.RFC1123, + "RFC1123Z": time.RFC1123Z, + "RFC3339": time.RFC3339, + "RFC3339Nano": time.RFC3339Nano, + "Kitchen": time.Kitchen, + } +) + +func formatDate(date time.Time, layout string, tz ...string) string { + if found := predefinedLayouts[layout]; found != "" { + layout = found + } else { + layout = time.RFC3339 + } + + if len(tz) > 0 { + if loc, err := time.LoadLocation(tz[0]); err == nil { + date = date.In(loc) + } else { + date = date.UTC() + } + } else { + date = date.UTC() + } + + return date.Format(layout) +} + +func parseDate(date, layout string) time.Time { + if found := predefinedLayouts[layout]; found != "" { + layout = found + } else { + layout = time.RFC3339 + } + + t, err := time.Parse(layout, date) + if err != nil { + return time.Time{} + } + + return t +} + +func now(add ...time.Duration) time.Time { + now := time.Now() + if len(add) == 0 { + return now + } + return now.Add(add[0]) +} + +func getRFC5988Link(links, rel string) string { + return "" +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go new file mode 100644 index 00000000000..bc9b4b5a55b --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go @@ -0,0 +1,182 @@ +package transforms + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +const logName = "httpjson.transforms" + +// Config represents the list of transforms. +type Config []*common.Config + +type Transforms struct { + List []Transform + log *logp.Logger +} + +type Transform interface { + Run(*Transformable) (*Transformable, error) + String() string +} + +type TargetType int + +const ( + TargetBody TargetType = iota + TargetCursor + TargetHeaders + TargetURLValue + TargetURLParams +) + +type ErrInvalidTarget struct { + target string +} + +func (err ErrInvalidTarget) Error() string { + return fmt.Sprintf("invalid target %q", err.target) +} + +type Transformable struct { + Headers http.Header + Body common.MapStr + URL url.URL + Cursor common.MapStr + LastEvent common.MapStr + LastResponse common.MapStr +} + +func NewEmptyTransformable() *Transformable { + return &Transformable{ + Headers: make(http.Header), + Body: make(common.MapStr), + Cursor: make(common.MapStr), + LastEvent: make(common.MapStr), + LastResponse: make(common.MapStr), + } +} + +type TargetInfo struct { + Type TargetType + Name string +} + +// NewList creates a new empty transform list. +// Additional processors can be added to the List field. +func NewList(log *logp.Logger) *Transforms { + if log == nil { + log = logp.NewLogger(logName) + } + return &Transforms{log: log} +} + +// New creates a list of transforms from a list of free user configurations. +func New(config Config, namespace string) (*Transforms, error) { + trans := NewList(nil) + + for _, tfConfig := range config { + if len(tfConfig.GetFields()) != 1 { + return nil, errors.Errorf( + "each transform must have exactly one action, but found %d actions (%v)", + len(tfConfig.GetFields()), + strings.Join(tfConfig.GetFields(), ","), + ) + } + + actionName := tfConfig.GetFields()[0] + cfg, err := tfConfig.Child(actionName, -1) + if err != nil { + return nil, err + } + + constructor, found := registeredTransforms.get(namespace, actionName) + if !found { + return nil, errors.Errorf("the transform %s does not exist. Valid transforms: %s", actionName, registeredTransforms.String()) + } + + cfg.PrintDebugf("Configure transform '%v' with:", actionName) + transform, err := constructor(cfg) + if err != nil { + return nil, err + } + + trans.Add(transform) + } + + if len(trans.List) > 0 { + trans.log.Debugf("Generated new transforms: %v", trans) + } + + return trans, nil +} + +func (trans *Transforms) Add(t Transform) { + if trans == nil { + return + } + trans.List = append(trans.List, t) +} + +// Run executes all transforms serially and returns the event and possibly +// an error. +func (trans *Transforms) Run(tr *Transformable) (*Transformable, error) { + var err error + for _, p := range trans.List { + tr, err = p.Run(tr) + if err != nil { + return tr, errors.Wrapf(err, "failed applying transform %v", tr) + } + } + return tr, nil +} + +func (trans Transforms) String() string { + var s []string + for _, p := range trans.List { + s = append(s, p.String()) + } + return strings.Join(s, ", ") +} + +func GetTargetInfo(t string) TargetInfo { + parts := strings.SplitN(t, ".", 2) + if len(parts) < 2 { + return TargetInfo{} + } + switch parts[0] { + case "url": + if parts[1] == "value" { + return TargetInfo{Type: TargetURLValue} + } + + paramParts := strings.SplitN(parts[1], ".", 2) + return TargetInfo{ + Type: TargetURLParams, + Name: paramParts[1], + } + case "headers": + return TargetInfo{ + Type: TargetHeaders, + Name: parts[1], + } + case "body": + return TargetInfo{ + Type: TargetBody, + Name: parts[1], + } + case "cursor": + return TargetInfo{ + Type: TargetCursor, + Name: parts[1], + } + } + return TargetInfo{} +} diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/v2/request.go new file mode 100644 index 00000000000..11865293344 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/request.go @@ -0,0 +1,155 @@ +package v2 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/append" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/set" +) + +const requestNamespace = "request" + +func registerRequestTransforms() { + transforms.RegisterTransform(requestNamespace, set.Name, set.New) + transforms.RegisterTransform(requestNamespace, append.Name, append.New) + transforms.RegisterTransform(requestNamespace, delete.Name, delete.New) +} + +type requestFactory struct { + url url.URL + method string + body *common.MapStr + transforms []transforms.Transform + user string + password string + log *logp.Logger +} + +func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { + // config validation already checked for errors here + ts, _ := transforms.New(config.Transforms, requestNamespace) + rf := &requestFactory{ + url: *config.URL.URL, + method: config.Method, + body: config.Body, + transforms: ts.List, + log: log, + } + if authConfig != nil && authConfig.Basic.isEnabled() { + rf.user = authConfig.Basic.User + rf.password = authConfig.Basic.Password + } + return rf +} + +func (rf *requestFactory) newRequest(ctx context.Context) (*http.Request, error) { + var err error + + trReq := transforms.NewEmptyTransformable() + + clonedURL, err := url.Parse(rf.url.String()) + if err != nil { + return nil, err + } + trReq.URL = *clonedURL + + if rf.body != nil { + trReq.Body = rf.body.Clone() + } + + for _, t := range rf.transforms { + trReq, err = t.Run(trReq) + if err != nil { + return nil, err + } + } + + var body []byte + if len(trReq.Body) > 0 { + switch rf.method { + case "POST": + body, err = json.Marshal(trReq.Body) + if err != nil { + return nil, err + } + default: + rf.log.Errorf("A body is set, but method is not POST. The body will be ignored.") + } + } + + req, err := http.NewRequest(rf.method, trReq.URL.String(), bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + + req = req.WithContext(ctx) + + req.Header = trReq.Headers + req.Header.Set("Accept", "application/json") + if rf.method == "POST" { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("User-Agent", userAgent) + + if rf.user != "" || rf.password != "" { + req.SetBasicAuth(rf.user, rf.password) + } + + return req, nil +} + +type requester struct { + log *logp.Logger + client *http.Client + requestFactory *requestFactory + responseProcessor *responseProcessor +} + +func newRequester(client *http.Client, requestFactory *requestFactory, responseProcessor *responseProcessor, log *logp.Logger) *requester { + return &requester{ + log: log, + client: client, + requestFactory: requestFactory, + responseProcessor: responseProcessor, + } +} + +func (r *requester) processRequest(ctx context.Context, publisher cursor.Publisher) error { + req, err := r.requestFactory.newRequest(ctx) + if err != nil { + return fmt.Errorf("failed to create http request: %w", err) + } + + resp, err := r.client.Do(req) + if err != nil { + return fmt.Errorf("failed to execute http client.Do: %w", err) + } + + events, err := r.responseProcessor.getEventsFromResponse(ctx, resp) + if err != nil { + return err + } + + for e := range events { + if e.failed() { + r.log.Errorf("failed to create event: %v", e.err) + continue + } + + if err := publisher.Publish(e.event, nil); err != nil { + return err + } + } + + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go new file mode 100644 index 00000000000..eeb374be683 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -0,0 +1,104 @@ +package v2 + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/append" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/set" +) + +const ( + responseNamespace = "response" +) + +func registerResponseTransforms() { + transforms.RegisterTransform(responseNamespace, set.Name, set.New) + transforms.RegisterTransform(responseNamespace, append.Name, append.New) + transforms.RegisterTransform(responseNamespace, delete.Name, delete.New) +} + +type responseProcessor struct { + log *logp.Logger + transforms []transforms.Transform +} + +func newResponseProcessor(config *responseConfig) *responseProcessor { + rp := &responseProcessor{} + if config == nil { + return rp + } + + tr, _ := transforms.New(config.Transforms, responseNamespace) + rp.transforms = tr.List + + return rp +} + +type maybeEvent struct { + event beat.Event + err error +} + +func (e maybeEvent) failed() bool { + return e.err != nil +} + +func (rp *responseProcessor) getEventsFromResponse(ctx context.Context, resp *http.Response) (<-chan maybeEvent, error) { + responseData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read http response: %w", err) + } + + var m common.MapStr + if err := json.Unmarshal(responseData, &m); err != nil { + return nil, err + } + + trResp := transforms.NewEmptyTransformable() + trResp.Body = m + trResp.Headers = resp.Header.Clone() + trResp.URL = *resp.Request.URL + + return rp.run(ctx, trResp), nil +} + +func (rp *responseProcessor) run(ctx context.Context, trResp *transforms.Transformable) <-chan maybeEvent { + ch := make(chan maybeEvent) + + go func() { + defer close(ch) + var err error + for _, tr := range rp.transforms { + select { + case <-ctx.Done(): + return + default: + } + + trResp, err = tr.Run(trResp) + if err != nil { + rp.log.Errorf("error running transform: %v", err) + continue + } + + b, err := json.Marshal(trResp.Body) + if err != nil { + ch <- maybeEvent{err: err} + continue + } + + ch <- maybeEvent{event: makeEvent(string(b))} + } + }() + + return ch +} From 2c37df62ef2c63d3d2d5b3a1c9250f695cbb118f Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 29 Oct 2020 14:58:41 +0100 Subject: [PATCH 05/35] Refactor and set up basic requester structure --- .../filebeat/input/default-inputs/inputs.go | 4 +- x-pack/filebeat/input/httpjson/v2/config.go | 5 +- .../input/httpjson/v2/config_request.go | 39 +++- .../input/httpjson/v2/config_response.go | 12 +- x-pack/filebeat/input/httpjson/v2/input.go | 34 +-- .../input/httpjson/v2/input_manager.go | 2 +- .../v2/internal/transforms/append/append.go | 92 --------- .../v2/internal/transforms/delete/delete.go | 72 ------- .../v2/internal/transforms/set/set.go | 92 --------- .../v2/internal/transforms/template.go | 53 ----- .../v2/internal/transforms/transform.go | 182 ---------------- .../filebeat/input/httpjson/v2/pagination.go | 26 +++ x-pack/filebeat/input/httpjson/v2/request.go | 120 ++++++----- x-pack/filebeat/input/httpjson/v2/response.go | 100 +-------- .../filebeat/input/httpjson/v2/transform.go | 157 ++++++++++++++ .../registry.go => transform_registry.go} | 20 +- .../input/httpjson/v2/transform_target.go | 64 ++++++ .../input/httpjson/v2/transfrom_append.go | 193 +++++++++++++++++ .../input/httpjson/v2/transfrom_delete.go | 173 ++++++++++++++++ .../input/httpjson/v2/transfrom_set.go | 194 ++++++++++++++++++ .../input/httpjson/v2/transfrom_split.go | 1 + .../template_funcs.go => value_tpl.go} | 55 ++++- 22 files changed, 995 insertions(+), 695 deletions(-) delete mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go delete mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go delete mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go delete mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go delete mode 100644 x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go create mode 100644 x-pack/filebeat/input/httpjson/v2/pagination.go create mode 100644 x-pack/filebeat/input/httpjson/v2/transform.go rename x-pack/filebeat/input/httpjson/v2/{internal/transforms/registry.go => transform_registry.go} (67%) create mode 100644 x-pack/filebeat/input/httpjson/v2/transform_target.go create mode 100644 x-pack/filebeat/input/httpjson/v2/transfrom_append.go create mode 100644 x-pack/filebeat/input/httpjson/v2/transfrom_delete.go create mode 100644 x-pack/filebeat/input/httpjson/v2/transfrom_set.go create mode 100644 x-pack/filebeat/input/httpjson/v2/transfrom_split.go rename x-pack/filebeat/input/httpjson/v2/{internal/transforms/template_funcs.go => value_tpl.go} (52%) diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 4779b452f1d..01e9b8ba137 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -12,7 +12,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" + httpjsonv2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" ) @@ -28,7 +28,7 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 return []v2.Plugin{ cloudfoundry.Plugin(), http_endpoint.Plugin(), - httpjson.Plugin(log, store), + httpjsonv2.Plugin(log, store), o365audit.Plugin(log, store), s3.Plugin(), } diff --git a/x-pack/filebeat/input/httpjson/v2/config.go b/x-pack/filebeat/input/httpjson/v2/config.go index 71ee0e96538..fec00aac041 100644 --- a/x-pack/filebeat/input/httpjson/v2/config.go +++ b/x-pack/filebeat/input/httpjson/v2/config.go @@ -20,11 +20,14 @@ func (c config) Validate() error { } func defaultConfig() config { + timeout := 30 * time.Second return config{ Interval: time.Minute, Auth: &authConfig{}, Request: &requestConfig{ - Method: "GET", + Timeout: &timeout, + Method: "GET", }, + Response: &responseConfig{}, } } diff --git a/x-pack/filebeat/input/httpjson/v2/config_request.go b/x-pack/filebeat/input/httpjson/v2/config_request.go index a0a96b67906..53e55299ba7 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/v2/config_request.go @@ -9,7 +9,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" ) type retryConfig struct { @@ -30,10 +29,31 @@ func (c retryConfig) Validate() error { return nil } +func (c retryConfig) getMaxAttempts() int { + if c.MaxAttempts == nil { + return 0 + } + return *c.MaxAttempts +} + +func (c retryConfig) getWaitMin() time.Duration { + if c.WaitMin == nil { + return 0 + } + return *c.WaitMin +} + +func (c retryConfig) getWaitMax() time.Duration { + if c.WaitMax == nil { + return 0 + } + return *c.WaitMax +} + type rateLimitConfig struct { - Limit *transforms.Template `config:"limit"` - Reset *transforms.Template `config:"reset"` - Remaining *transforms.Template `config:"remaining"` + Limit *valueTpl `config:"limit"` + Reset *valueTpl `config:"reset"` + Remaining *valueTpl `config:"remaining"` } func (c rateLimitConfig) Validate() error { @@ -67,7 +87,14 @@ type requestConfig struct { SSL *tlscommon.Config `config:"ssl"` Retry retryConfig `config:"retry"` RateLimit *rateLimitConfig `config:"rate_limit"` - Transforms transforms.Config `config:"transforms"` + Transforms transformsConfig `config:"transforms"` +} + +func (c requestConfig) getTimeout() time.Duration { + if c.Timeout == nil { + return 0 + } + return *c.Timeout } func (c *requestConfig) Validate() error { @@ -86,7 +113,7 @@ func (c *requestConfig) Validate() error { return errors.New("timeout must be greater than 0") } - if _, err := transforms.New(c.Transforms, requestNamespace); err != nil { + if _, err := newRequestTransformsFromConfig(c.Transforms); err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/v2/config_response.go b/x-pack/filebeat/input/httpjson/v2/config_response.go index e174c2addc8..86c5d22d621 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/v2/config_response.go @@ -1,15 +1,15 @@ package v2 -import ( - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" -) - type responseConfig struct { - Transforms transforms.Config `config:"transforms"` + Transforms transformsConfig `config:"transforms"` + Pagination transformsConfig `config:"pagination"` } func (c *responseConfig) Validate() error { - if _, err := transforms.New(c.Transforms, responseNamespace); err != nil { + if _, err := newResponseTransformsFromConfig(c.Transforms); err != nil { + return err + } + if _, err := newPaginationTransformsFromConfig(c.Transforms); err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go index 236a01b9711..51dcdbb02d3 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -10,7 +10,6 @@ import ( "net" "net/http" "net/url" - "reflect" "time" "github.com/hashicorp/go-retryablehttp" @@ -71,6 +70,7 @@ func Plugin(log *logp.Logger, store cursor.StateStore) v2.Plugin { registerRequestTransforms() registerResponseTransforms() + registerPaginationTransforms() return v2.Plugin{ Name: inputName, @@ -132,24 +132,21 @@ func run( } requestFactory := newRequestFactory(config.Request, config.Auth, log) - responseProcessor := newResponseProcessor(config.Response) - - requester := newRequester(httpClient, requestFactory, responseProcessor, log) + requester := newRequester(httpClient, requestFactory, log) + // loadContextFromCursor + trCtx := transformContext{} err = timed.Periodic(stdCtx, config.Interval, func() error { log.Info("Process another repeated request.") - err := requester.processRequest(stdCtx, publisher) - if err == nil { - return nil + if err := requester.doRequest(stdCtx, trCtx, publisher); err != nil { + log.Errorf("Error while processing http request: %v", err) } if stdCtx.Err() != nil { return err } - log.Errorf("Error while processing http request: %v", err) - return nil }) @@ -159,7 +156,7 @@ func run( } func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig) (*http.Client, error) { - timeout := getValFromPtr(config.Request.Timeout, time.Duration(0)).(time.Duration) + timeout := config.Request.getTimeout() // Make retryable HTTP client client := &retryablehttp.Client{ @@ -174,9 +171,9 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC Timeout: timeout, }, Logger: newRetryLogger(), - RetryWaitMin: getValFromPtr(config.Request.Retry.WaitMin, time.Duration(0)).(time.Duration), - RetryWaitMax: getValFromPtr(config.Request.Retry.WaitMax, time.Duration(0)).(time.Duration), - RetryMax: getValFromPtr(config.Request.Retry.MaxAttempts, 0).(int), + RetryWaitMin: config.Request.Retry.getWaitMin(), + RetryWaitMax: config.Request.Retry.getWaitMax(), + RetryMax: config.Request.Retry.getMaxAttempts(), CheckRetry: retryablehttp.DefaultRetryPolicy, Backoff: retryablehttp.DefaultBackoff, } @@ -188,17 +185,6 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC return client.StandardClient(), nil } -func getValFromPtr(ptr, defaultVal interface{}) interface{} { - switch { - case reflect.ValueOf(ptr).Kind() != reflect.Ptr, - !reflect.ValueOf(ptr).IsValid(), - reflect.ValueOf(ptr).IsNil(): - return defaultVal - } - - return reflect.Indirect(reflect.ValueOf(ptr)) -} - func makeEvent(body string) beat.Event { now := timeNow() fields := common.MapStr{ diff --git a/x-pack/filebeat/input/httpjson/v2/input_manager.go b/x-pack/filebeat/input/httpjson/v2/input_manager.go index 971740ccab8..473e33fa7c1 100644 --- a/x-pack/filebeat/input/httpjson/v2/input_manager.go +++ b/x-pack/filebeat/input/httpjson/v2/input_manager.go @@ -29,7 +29,7 @@ func (m inputManager) Init(grp unison.Group, mode v2.Mode) error { // Create creates a cursor input manager if the config has a date cursor set up, // otherwise it creates a stateless input manager. func (m inputManager) Create(cfg *common.Config) (v2.Input, error) { - var config config + config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err } diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go deleted file mode 100644 index 1a64e01a649..00000000000 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/append/append.go +++ /dev/null @@ -1,92 +0,0 @@ -package append - -import ( - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" - "github.com/pkg/errors" -) - -const Name = "append" - -type config struct { - Target string `config:"target"` - Value *transforms.Template `config:"value"` - Default string `config:"default"` -} - -type appendTransform struct { - targetInfo transforms.TargetInfo - value *transforms.Template - defaultValue string - - run func(tr *transforms.Transformable, key, val string) error -} - -func New(cfg *common.Config) (transforms.Transform, error) { - c := &config{} - if err := cfg.Unpack(c); err != nil { - return nil, errors.Wrap(err, "fail to unpack the append configuration") - } - app := &appendTransform{ - targetInfo: transforms.GetTargetInfo(c.Target), - value: c.Value, - defaultValue: c.Default, - } - - switch app.targetInfo.Type { - // case transforms.TargetCursor: - case transforms.TargetBody: - app.run = runBody - case transforms.TargetHeaders: - app.run = runHeader - case transforms.TargetURLParams: - app.run = runURLParams - case transforms.TargetURLValue: - return nil, errors.New("can't append to url.value") - default: - return nil, errors.New("unknown target type") - } - - return app, nil -} - -func (appendTransform) String() string { return Name } - -func (app *appendTransform) Run(tr *transforms.Transformable) (*transforms.Transformable, error) { - value := app.value.Execute(tr, app.defaultValue) - return tr, app.run(tr, app.targetInfo.Name, value) -} - -func appendToCommonMap(m common.MapStr, key, val string) error { - var value interface{} = val - if found, _ := m.HasKey(key); found { - prev, _ := m.GetValue(key) - switch t := prev.(type) { - case []string: - value = append(t, val) - case []interface{}: - value = append(t, val) - default: - value = []interface{}{prev, val} - } - - } - if _, err := m.Put(key, value); err != nil { - return err - } - return nil -} - -func runBody(tr *transforms.Transformable, key, value string) error { - return appendToCommonMap(tr.Body, key, value) -} - -func runHeader(tr *transforms.Transformable, key, value string) error { - tr.Headers.Add(key, value) - return nil -} - -func runURLParams(tr *transforms.Transformable, key, value string) error { - tr.URL.Query().Add(key, value) - return nil -} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go deleted file mode 100644 index 6b41cb0420b..00000000000 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete/delete.go +++ /dev/null @@ -1,72 +0,0 @@ -package delete - -import ( - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" - "github.com/pkg/errors" -) - -const Name = "delete" - -type config struct { - Target string `config:"target"` -} - -type delete struct { - targetInfo transforms.TargetInfo - - run func(tr *transforms.Transformable, key string) error -} - -func New(cfg *common.Config) (transforms.Transform, error) { - c := &config{} - if err := cfg.Unpack(c); err != nil { - return nil, errors.Wrap(err, "fail to unpack the set configuration") - } - delete := &delete{ - targetInfo: transforms.GetTargetInfo(c.Target), - } - - switch delete.targetInfo.Type { - // case transforms.TargetCursor: - case transforms.TargetBody: - delete.run = runBody - case transforms.TargetHeaders: - delete.run = runHeader - case transforms.TargetURLParams: - delete.run = runURLParams - case transforms.TargetURLValue: - return nil, errors.New("can't append to url.value") - default: - return nil, errors.New("unknown target type") - } - - return delete, nil -} - -func (delete) String() string { return Name } - -func (delete *delete) Run(tr *transforms.Transformable) (*transforms.Transformable, error) { - return tr, delete.run(tr, delete.targetInfo.Name) -} - -func deleteFromCommonMap(m common.MapStr, key string) error { - if err := m.Delete(key); err != common.ErrKeyNotFound { - return err - } - return nil -} - -func runBody(tr *transforms.Transformable, key string) error { - return deleteFromCommonMap(tr.Body, key) -} - -func runHeader(tr *transforms.Transformable, key string) error { - tr.Headers.Del(key) - return nil -} - -func runURLParams(tr *transforms.Transformable, key string) error { - tr.URL.Query().Del(key) - return nil -} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go deleted file mode 100644 index 3c23ab88758..00000000000 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/set/set.go +++ /dev/null @@ -1,92 +0,0 @@ -package set - -import ( - "net/url" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" - "github.com/pkg/errors" -) - -const Name = "set" - -type config struct { - Target string `config:"target"` - Value *transforms.Template `config:"value"` - Default string `config:"default"` -} - -type set struct { - targetInfo transforms.TargetInfo - value *transforms.Template - defaultValue string - - run func(tr *transforms.Transformable, key, val string) error -} - -func New(cfg *common.Config) (transforms.Transform, error) { - c := &config{} - if err := cfg.Unpack(c); err != nil { - return nil, errors.Wrap(err, "fail to unpack the set configuration") - } - set := &set{ - targetInfo: transforms.GetTargetInfo(c.Target), - value: c.Value, - defaultValue: c.Default, - } - - switch set.targetInfo.Type { - // case transforms.TargetCursor: - case transforms.TargetBody: - set.run = runBody - case transforms.TargetHeaders: - set.run = runHeader - case transforms.TargetURLValue: - set.run = runURLValue - case transforms.TargetURLParams: - set.run = runURLParams - default: - return nil, errors.New("unknown target type") - } - - return set, nil -} - -func (set) String() string { return Name } - -func (set *set) Run(tr *transforms.Transformable) (*transforms.Transformable, error) { - value := set.value.Execute(tr, set.defaultValue) - return tr, set.run(tr, set.targetInfo.Name, value) -} - -func setToCommonMap(m common.MapStr, key, val string) error { - if _, err := m.Put(key, val); err != nil { - return err - } - return nil -} - -func runBody(tr *transforms.Transformable, key, value string) error { - return setToCommonMap(tr.Body, key, value) -} - -func runHeader(tr *transforms.Transformable, key, value string) error { - tr.Headers.Add(key, value) - return nil -} - -func runURLParams(tr *transforms.Transformable, key, value string) error { - tr.URL.Query().Add(key, value) - return nil -} - -func runURLValue(tr *transforms.Transformable, _, value string) error { - query := tr.URL.Query().Encode() - url, err := url.Parse(value) - if err != nil { - return err - } - url.RawQuery = query - tr.URL = *url - return nil -} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go deleted file mode 100644 index ab664af17d2..00000000000 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/template.go +++ /dev/null @@ -1,53 +0,0 @@ -package transforms - -import ( - "bytes" - "html/template" -) - -type Template struct { - *template.Template -} - -func (t *Template) Unpack(in string) error { - tpl, err := template.New(""). - Option("missingkey=error"). - Funcs(template.FuncMap{ - "now": now, - "formatDate": formatDate, - "parseDate": parseDate, - "getRFC5988Link": getRFC5988Link, - }). - Parse(in) - if err != nil { - return err - } - - *t = Template{Template: tpl} - - return nil -} - -func (t *Template) Execute(tr *Transformable, defaultVal string) (val string) { - defer func() { - if r := recover(); r != nil { - // really ugly - val = defaultVal - } - }() - - buf := new(bytes.Buffer) - data := map[string]interface{}{ - "header": tr.Headers.Clone(), - "body": tr.Body.Clone(), - "url.value": tr.URL.String(), - "url.params": tr.URL.Query(), - // "cursor": tr.Cursor.Clone(), - // "last_event": tr.LastEvent, - // "last_response": tr.LastResponse.Clone(), - } - if err := t.Template.Execute(buf, data); err != nil { - return defaultVal - } - return buf.String() -} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go b/x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go deleted file mode 100644 index bc9b4b5a55b..00000000000 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/transform.go +++ /dev/null @@ -1,182 +0,0 @@ -package transforms - -import ( - "fmt" - "net/http" - "net/url" - "strings" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -const logName = "httpjson.transforms" - -// Config represents the list of transforms. -type Config []*common.Config - -type Transforms struct { - List []Transform - log *logp.Logger -} - -type Transform interface { - Run(*Transformable) (*Transformable, error) - String() string -} - -type TargetType int - -const ( - TargetBody TargetType = iota - TargetCursor - TargetHeaders - TargetURLValue - TargetURLParams -) - -type ErrInvalidTarget struct { - target string -} - -func (err ErrInvalidTarget) Error() string { - return fmt.Sprintf("invalid target %q", err.target) -} - -type Transformable struct { - Headers http.Header - Body common.MapStr - URL url.URL - Cursor common.MapStr - LastEvent common.MapStr - LastResponse common.MapStr -} - -func NewEmptyTransformable() *Transformable { - return &Transformable{ - Headers: make(http.Header), - Body: make(common.MapStr), - Cursor: make(common.MapStr), - LastEvent: make(common.MapStr), - LastResponse: make(common.MapStr), - } -} - -type TargetInfo struct { - Type TargetType - Name string -} - -// NewList creates a new empty transform list. -// Additional processors can be added to the List field. -func NewList(log *logp.Logger) *Transforms { - if log == nil { - log = logp.NewLogger(logName) - } - return &Transforms{log: log} -} - -// New creates a list of transforms from a list of free user configurations. -func New(config Config, namespace string) (*Transforms, error) { - trans := NewList(nil) - - for _, tfConfig := range config { - if len(tfConfig.GetFields()) != 1 { - return nil, errors.Errorf( - "each transform must have exactly one action, but found %d actions (%v)", - len(tfConfig.GetFields()), - strings.Join(tfConfig.GetFields(), ","), - ) - } - - actionName := tfConfig.GetFields()[0] - cfg, err := tfConfig.Child(actionName, -1) - if err != nil { - return nil, err - } - - constructor, found := registeredTransforms.get(namespace, actionName) - if !found { - return nil, errors.Errorf("the transform %s does not exist. Valid transforms: %s", actionName, registeredTransforms.String()) - } - - cfg.PrintDebugf("Configure transform '%v' with:", actionName) - transform, err := constructor(cfg) - if err != nil { - return nil, err - } - - trans.Add(transform) - } - - if len(trans.List) > 0 { - trans.log.Debugf("Generated new transforms: %v", trans) - } - - return trans, nil -} - -func (trans *Transforms) Add(t Transform) { - if trans == nil { - return - } - trans.List = append(trans.List, t) -} - -// Run executes all transforms serially and returns the event and possibly -// an error. -func (trans *Transforms) Run(tr *Transformable) (*Transformable, error) { - var err error - for _, p := range trans.List { - tr, err = p.Run(tr) - if err != nil { - return tr, errors.Wrapf(err, "failed applying transform %v", tr) - } - } - return tr, nil -} - -func (trans Transforms) String() string { - var s []string - for _, p := range trans.List { - s = append(s, p.String()) - } - return strings.Join(s, ", ") -} - -func GetTargetInfo(t string) TargetInfo { - parts := strings.SplitN(t, ".", 2) - if len(parts) < 2 { - return TargetInfo{} - } - switch parts[0] { - case "url": - if parts[1] == "value" { - return TargetInfo{Type: TargetURLValue} - } - - paramParts := strings.SplitN(parts[1], ".", 2) - return TargetInfo{ - Type: TargetURLParams, - Name: paramParts[1], - } - case "headers": - return TargetInfo{ - Type: TargetHeaders, - Name: parts[1], - } - case "body": - return TargetInfo{ - Type: TargetBody, - Name: parts[1], - } - case "cursor": - return TargetInfo{ - Type: TargetCursor, - Name: parts[1], - } - } - return TargetInfo{} -} diff --git a/x-pack/filebeat/input/httpjson/v2/pagination.go b/x-pack/filebeat/input/httpjson/v2/pagination.go new file mode 100644 index 00000000000..afdd4f66e18 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/pagination.go @@ -0,0 +1,26 @@ +package v2 + +import ( + "net/http" + "net/url" + + "github.com/elastic/beats/v7/libbeat/common" +) + +const paginationNamespace = "pagination" + +func registerPaginationTransforms() { + registerTransform(paginationNamespace, appendName, newAppendPagination) + registerTransform(paginationNamespace, deleteName, newDeletePagination) + registerTransform(paginationNamespace, setName, newSetPagination) +} + +type pagination struct { + body common.MapStr + header http.Header + url *url.URL +} + +func (p *pagination) nextPageRequest() (*http.Request, error) { + return nil, nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/v2/request.go index 11865293344..a636261c27f 100644 --- a/x-pack/filebeat/input/httpjson/v2/request.go +++ b/x-pack/filebeat/input/httpjson/v2/request.go @@ -5,31 +5,60 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/append" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/set" ) const requestNamespace = "request" func registerRequestTransforms() { - transforms.RegisterTransform(requestNamespace, set.Name, set.New) - transforms.RegisterTransform(requestNamespace, append.Name, append.New) - transforms.RegisterTransform(requestNamespace, delete.Name, delete.New) + registerTransform(requestNamespace, appendName, newAppendRequest) + registerTransform(requestNamespace, deleteName, newDeleteRequest) + registerTransform(requestNamespace, setName, newSetRequest) +} + +type request struct { + body common.MapStr + header http.Header + url *url.URL +} + +func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []requestTransform) (*request, error) { + req := &request{ + body: common.MapStr{}, + header: http.Header{}, + } + + clonedURL, err := url.Parse(url.String()) + if err != nil { + return nil, err + } + req.url = clonedURL + + if body != nil { + req.body.DeepUpdate(*body) + } + + for _, t := range trs { + req, err = t.run(ctx, req) + if err != nil { + return nil, err + } + } + + return req, nil } type requestFactory struct { url url.URL method string body *common.MapStr - transforms []transforms.Transform + transforms []requestTransform user string password string log *logp.Logger @@ -37,12 +66,12 @@ type requestFactory struct { func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { // config validation already checked for errors here - ts, _ := transforms.New(config.Transforms, requestNamespace) + ts, _ := newRequestTransformsFromConfig(config.Transforms) rf := &requestFactory{ url: *config.URL.URL, method: config.Method, body: config.Body, - transforms: ts.List, + transforms: ts, log: log, } if authConfig != nil && authConfig.Basic.isEnabled() { @@ -52,33 +81,17 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. return rf } -func (rf *requestFactory) newRequest(ctx context.Context) (*http.Request, error) { - var err error - - trReq := transforms.NewEmptyTransformable() - - clonedURL, err := url.Parse(rf.url.String()) +func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transformContext) (*http.Request, error) { + trReq, err := newRequest(trCtx, rf.body, rf.url, rf.transforms) if err != nil { return nil, err } - trReq.URL = *clonedURL - - if rf.body != nil { - trReq.Body = rf.body.Clone() - } - - for _, t := range rf.transforms { - trReq, err = t.Run(trReq) - if err != nil { - return nil, err - } - } var body []byte - if len(trReq.Body) > 0 { + if len(trReq.body) > 0 { switch rf.method { case "POST": - body, err = json.Marshal(trReq.Body) + body, err = json.Marshal(trReq.body) if err != nil { return nil, err } @@ -87,19 +100,19 @@ func (rf *requestFactory) newRequest(ctx context.Context) (*http.Request, error) } } - req, err := http.NewRequest(rf.method, trReq.URL.String(), bytes.NewBuffer(body)) + req, err := http.NewRequest(rf.method, trReq.url.String(), bytes.NewBuffer(body)) if err != nil { return nil, err } - req = req.WithContext(ctx) + req = req.WithContext(stdCtx) - req.Header = trReq.Headers + req.Header = trReq.header req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", userAgent) if rf.method == "POST" { req.Header.Set("Content-Type", "application/json") } - req.Header.Set("User-Agent", userAgent) if rf.user != "" || rf.password != "" { req.SetBasicAuth(rf.user, rf.password) @@ -109,23 +122,21 @@ func (rf *requestFactory) newRequest(ctx context.Context) (*http.Request, error) } type requester struct { - log *logp.Logger - client *http.Client - requestFactory *requestFactory - responseProcessor *responseProcessor + log *logp.Logger + client *http.Client + requestFactory *requestFactory } -func newRequester(client *http.Client, requestFactory *requestFactory, responseProcessor *responseProcessor, log *logp.Logger) *requester { +func newRequester(client *http.Client, requestFactory *requestFactory, log *logp.Logger) *requester { return &requester{ - log: log, - client: client, - requestFactory: requestFactory, - responseProcessor: responseProcessor, + log: log, + client: client, + requestFactory: requestFactory, } } -func (r *requester) processRequest(ctx context.Context, publisher cursor.Publisher) error { - req, err := r.requestFactory.newRequest(ctx) +func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, publisher cursor.Publisher) error { + req, err := r.requestFactory.newHTTPRequest(stdCtx, trCtx) if err != nil { return fmt.Errorf("failed to create http request: %w", err) } @@ -134,22 +145,9 @@ func (r *requester) processRequest(ctx context.Context, publisher cursor.Publish if err != nil { return fmt.Errorf("failed to execute http client.Do: %w", err) } + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() - events, err := r.responseProcessor.getEventsFromResponse(ctx, resp) - if err != nil { - return err - } - - for e := range events { - if e.failed() { - r.log.Errorf("failed to create event: %v", e.err) - continue - } - - if err := publisher.Publish(e.event, nil); err != nil { - return err - } - } - + fmt.Println(string(body)) return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index eeb374be683..6cab31c8e43 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -1,104 +1,22 @@ package v2 import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" "net/http" + "net/url" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/append" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/delete" - "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2/internal/transforms/set" ) -const ( - responseNamespace = "response" -) +const responseNamespace = "response" func registerResponseTransforms() { - transforms.RegisterTransform(responseNamespace, set.Name, set.New) - transforms.RegisterTransform(responseNamespace, append.Name, append.New) - transforms.RegisterTransform(responseNamespace, delete.Name, delete.New) -} - -type responseProcessor struct { - log *logp.Logger - transforms []transforms.Transform -} - -func newResponseProcessor(config *responseConfig) *responseProcessor { - rp := &responseProcessor{} - if config == nil { - return rp - } - - tr, _ := transforms.New(config.Transforms, responseNamespace) - rp.transforms = tr.List - - return rp -} - -type maybeEvent struct { - event beat.Event - err error -} - -func (e maybeEvent) failed() bool { - return e.err != nil + registerTransform(responseNamespace, appendName, newAppendResponse) + registerTransform(responseNamespace, deleteName, newDeleteResponse) + registerTransform(responseNamespace, setName, newSetResponse) } -func (rp *responseProcessor) getEventsFromResponse(ctx context.Context, resp *http.Response) (<-chan maybeEvent, error) { - responseData, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read http response: %w", err) - } - - var m common.MapStr - if err := json.Unmarshal(responseData, &m); err != nil { - return nil, err - } - - trResp := transforms.NewEmptyTransformable() - trResp.Body = m - trResp.Headers = resp.Header.Clone() - trResp.URL = *resp.Request.URL - - return rp.run(ctx, trResp), nil -} - -func (rp *responseProcessor) run(ctx context.Context, trResp *transforms.Transformable) <-chan maybeEvent { - ch := make(chan maybeEvent) - - go func() { - defer close(ch) - var err error - for _, tr := range rp.transforms { - select { - case <-ctx.Done(): - return - default: - } - - trResp, err = tr.Run(trResp) - if err != nil { - rp.log.Errorf("error running transform: %v", err) - continue - } - - b, err := json.Marshal(trResp.Body) - if err != nil { - ch <- maybeEvent{err: err} - continue - } - - ch <- maybeEvent{event: makeEvent(string(b))} - } - }() - - return ch +type response struct { + body common.MapStr + header http.Header + url *url.URL } diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go new file mode 100644 index 00000000000..6e630339b66 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -0,0 +1,157 @@ +package v2 + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/pkg/errors" +) + +const logName = "httpjson.transforms" + +type transformsConfig []*common.Config + +type transforms []transform + +type transformContext struct { + cursor common.MapStr + lastEvent common.MapStr + lastResponse common.MapStr +} + +type transformable struct { + body common.MapStr + header http.Header + url *url.URL +} + +type transform interface { + transformName() string +} + +type requestTransform interface { + transform + run(transformContext, *request) (*request, error) +} + +type responseTransform interface { + transform + run(transformContext, *response) (*response, error) +} + +type paginationTransform interface { + transform + run(transformContext, *pagination) (*pagination, error) +} + +type splitTransform interface { + transform + run(transformContext, *response, []beat.Event) ([]beat.Event, error) +} + +type cursorTransform interface { + transform + run(transformContext, common.MapStr) (common.MapStr, error) +} + +// newTransformsFromConfig creates a list of transforms from a list of free user configurations. +func newTransformsFromConfig(config transformsConfig, namespace string) (transforms, error) { + var trans transforms + + for _, tfConfig := range config { + if len(tfConfig.GetFields()) != 1 { + return nil, errors.Errorf( + "each transform must have exactly one action, but found %d actions (%v)", + len(tfConfig.GetFields()), + strings.Join(tfConfig.GetFields(), ","), + ) + } + + actionName := tfConfig.GetFields()[0] + cfg, err := tfConfig.Child(actionName, -1) + if err != nil { + return nil, err + } + + constructor, found := registeredTransforms.get(namespace, actionName) + if !found { + return nil, errors.Errorf("the transform %s does not exist. Valid transforms: %s", actionName, registeredTransforms.String()) + } + + cfg.PrintDebugf("Configure transform '%v' with:", actionName) + transform, err := constructor(cfg) + if err != nil { + return nil, err + } + + trans = append(trans, transform) + } + + return trans, nil +} + +func newRequestTransformsFromConfig(config transformsConfig) ([]requestTransform, error) { + ts, err := newTransformsFromConfig(config, requestNamespace) + if err != nil { + return nil, err + } + + var rts []requestTransform + for _, t := range ts { + rt, ok := t.(requestTransform) + if !ok { + return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), requestNamespace) + } + rts = append(rts, rt) + } + + return rts, nil +} + +func newResponseTransformsFromConfig(config transformsConfig) ([]responseTransform, error) { + ts, err := newTransformsFromConfig(config, responseNamespace) + if err != nil { + return nil, err + } + + var rts []responseTransform + for _, t := range ts { + rt, ok := t.(responseTransform) + if !ok { + return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), responseNamespace) + } + rts = append(rts, rt) + } + + return rts, nil +} + +func newPaginationTransformsFromConfig(config transformsConfig) ([]paginationTransform, error) { + ts, err := newTransformsFromConfig(config, paginationNamespace) + if err != nil { + return nil, err + } + + var rts []paginationTransform + for _, t := range ts { + rt, ok := t.(paginationTransform) + if !ok { + return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), paginationNamespace) + } + rts = append(rts, rt) + } + + return rts, nil +} + +func (trans transforms) String() string { + var s []string + for _, p := range trans { + s = append(s, p.transformName()) + } + return strings.Join(s, ", ") +} diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go b/x-pack/filebeat/input/httpjson/v2/transform_registry.go similarity index 67% rename from x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go rename to x-pack/filebeat/input/httpjson/v2/transform_registry.go index 77ffa8203cc..d75e7faca0e 100644 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/registry.go +++ b/x-pack/filebeat/input/httpjson/v2/transform_registry.go @@ -1,4 +1,4 @@ -package transforms +package v2 import ( "errors" @@ -9,26 +9,26 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -type Constructor func(config *common.Config) (Transform, error) +type constructor func(config *common.Config) (transform, error) var registeredTransforms = newRegistry() type registry struct { - namespaces map[string]map[string]Constructor + namespaces map[string]map[string]constructor } func newRegistry() *registry { - return ®istry{namespaces: make(map[string]map[string]Constructor)} + return ®istry{namespaces: make(map[string]map[string]constructor)} } -func (reg *registry) register(namespace, transform string, constructor Constructor) error { - if constructor == nil { +func (reg *registry) register(namespace, transform string, cons constructor) error { + if cons == nil { return errors.New("constructor can't be nil") } m, found := reg.namespaces[namespace] if !found { - reg.namespaces[namespace] = make(map[string]Constructor) + reg.namespaces[namespace] = make(map[string]constructor) m = reg.namespaces[namespace] } @@ -36,7 +36,7 @@ func (reg *registry) register(namespace, transform string, constructor Construct return errors.New("already registered") } - m[transform] = constructor + m[transform] = cons return nil } @@ -58,7 +58,7 @@ func (reg registry) String() string { return str } -func (reg registry) get(namespace, transform string) (Constructor, bool) { +func (reg registry) get(namespace, transform string) (constructor, bool) { m, found := reg.namespaces[namespace] if !found { return nil, false @@ -67,7 +67,7 @@ func (reg registry) get(namespace, transform string) (Constructor, bool) { return c, found } -func RegisterTransform(namespace, transform string, constructor Constructor) { +func registerTransform(namespace, transform string, constructor constructor) { logp.L().Named(logName).Debugf("Register transform %s:%s", namespace, transform) err := registeredTransforms.register(namespace, transform, constructor) diff --git a/x-pack/filebeat/input/httpjson/v2/transform_target.go b/x-pack/filebeat/input/httpjson/v2/transform_target.go new file mode 100644 index 00000000000..2741284c8fd --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transform_target.go @@ -0,0 +1,64 @@ +package v2 + +import ( + "fmt" + "strings" +) + +type targetType string + +const ( + targetBody targetType = "body" + targetCursor targetType = "cursor" + targetHeader targetType = "header" + targetURLValue targetType = "url.value" + targetURLParams targetType = "url.params" +) + +type errInvalidTarget struct { + target string +} + +func (err errInvalidTarget) Error() string { + return fmt.Sprintf("invalid target: %s", err.target) +} + +type targetInfo struct { + Type targetType + Name string +} + +func getTargetInfo(t string) (targetInfo, error) { + parts := strings.SplitN(t, ".", 2) + if len(parts) < 2 { + return targetInfo{}, errInvalidTarget{t} + } + switch parts[0] { + case "url": + if parts[1] == "value" { + return targetInfo{Type: targetURLValue}, nil + } + + paramParts := strings.SplitN(parts[1], ".", 2) + return targetInfo{ + Type: targetURLParams, + Name: paramParts[1], + }, nil + case "header": + return targetInfo{ + Type: targetHeader, + Name: parts[1], + }, nil + case "body": + return targetInfo{ + Type: targetBody, + Name: parts[1], + }, nil + case "cursor": + return targetInfo{ + Type: targetCursor, + Name: parts[1], + }, nil + } + return targetInfo{}, errInvalidTarget{t} +} diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go new file mode 100644 index 00000000000..20243c00f92 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go @@ -0,0 +1,193 @@ +package v2 + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/pkg/errors" +) + +const appendName = "append" + +var ( + _ requestTransform = &appendRequest{} + _ responseTransform = &appendResponse{} + _ paginationTransform = &appendPagination{} +) + +type appendConfig struct { + Target string `config:"target"` + Value *valueTpl `config:"value"` + Default string `config:"default"` +} + +type appendt struct { + targetInfo targetInfo + value *valueTpl + defaultValue string + + run func(ctx transformContext, transformable *transformable, key, val string) error +} + +func (appendt) transformName() string { return appendName } + +type appendRequest struct { + appendt +} + +type appendResponse struct { + appendt +} + +type appendPagination struct { + appendt +} + +func newAppendRequest(cfg *common.Config) (transform, error) { + append, err := newAppend(cfg) + if err != nil { + return nil, err + } + + switch append.targetInfo.Type { + case targetBody: + append.run = appendBody + case targetHeader: + append.run = appendHeader + case targetURLParams: + append.run = appendURLParams + default: + return nil, fmt.Errorf("invalid target type: %s", append.targetInfo.Type) + } + + return &appendRequest{appendt: append}, nil +} + +func (appendReq *appendRequest) run(ctx transformContext, req *request) (*request, error) { + transformable := &transformable{ + body: req.body, + header: req.header, + url: req.url, + } + if err := appendReq.appendt.runAppend(ctx, transformable); err != nil { + return nil, err + } + return req, nil +} + +func newAppendResponse(cfg *common.Config) (transform, error) { + append, err := newAppend(cfg) + if err != nil { + return nil, err + } + + switch append.targetInfo.Type { + case targetBody: + append.run = appendBody + default: + return nil, fmt.Errorf("invalid target type: %s", append.targetInfo.Type) + } + + return &appendResponse{appendt: append}, nil +} + +func (appendRes *appendResponse) run(ctx transformContext, res *response) (*response, error) { + transformable := &transformable{ + body: res.body, + header: res.header, + url: res.url, + } + if err := appendRes.appendt.runAppend(ctx, transformable); err != nil { + return nil, err + } + return res, nil +} + +func newAppendPagination(cfg *common.Config) (transform, error) { + append, err := newAppend(cfg) + if err != nil { + return nil, err + } + + switch append.targetInfo.Type { + case targetBody: + append.run = appendBody + case targetHeader: + append.run = appendHeader + case targetURLParams: + append.run = appendURLParams + default: + return nil, fmt.Errorf("invalid target type: %s", append.targetInfo.Type) + } + + return &appendPagination{appendt: append}, nil +} + +func (appendPag *appendPagination) run(ctx transformContext, pag *pagination) (*pagination, error) { + transformable := &transformable{ + body: pag.body, + header: pag.header, + url: pag.url, + } + if err := appendPag.appendt.runAppend(ctx, transformable); err != nil { + return nil, err + } + return pag, nil +} + +func newAppend(cfg *common.Config) (appendt, error) { + c := &appendConfig{} + if err := cfg.Unpack(c); err != nil { + return appendt{}, errors.Wrap(err, "fail to unpack the append configuration") + } + + ti, err := getTargetInfo(c.Target) + if err != nil { + return appendt{}, err + } + + return appendt{ + targetInfo: ti, + value: c.Value, + defaultValue: c.Default, + }, nil +} + +func (append *appendt) runAppend(ctx transformContext, transformable *transformable) error { + value := append.value.Execute(append.defaultValue) + return append.run(ctx, transformable, append.targetInfo.Name, value) +} + +func appendToCommonMap(m common.MapStr, key, val string) error { + var value interface{} = val + if found, _ := m.HasKey(key); found { + prev, _ := m.GetValue(key) + switch t := prev.(type) { + case []string: + value = append(t, val) + case []interface{}: + value = append(t, val) + default: + value = []interface{}{prev, val} + } + + } + if _, err := m.Put(key, value); err != nil { + return err + } + return nil +} + +func appendBody(ctx transformContext, transformable *transformable, key, value string) error { + return appendToCommonMap(transformable.body, key, value) +} + +func appendHeader(ctx transformContext, transformable *transformable, key, value string) error { + transformable.header.Add(key, value) + return nil +} + +func appendURLParams(ctx transformContext, transformable *transformable, key, value string) error { + transformable.url.Query().Add(key, value) + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go b/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go new file mode 100644 index 00000000000..4a64556d228 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go @@ -0,0 +1,173 @@ +package v2 + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/pkg/errors" +) + +const deleteName = "delete" + +var ( + _ requestTransform = &deleteRequest{} + _ responseTransform = &deleteResponse{} + _ paginationTransform = &deletePagination{} +) + +type deleteConfig struct { + Target string `config:"target"` +} + +type delete struct { + targetInfo targetInfo + + run func(ctx transformContext, transformable *transformable, key string) error +} + +func (delete) transformName() string { return deleteName } + +type deleteRequest struct { + delete +} + +type deleteResponse struct { + delete +} + +type deletePagination struct { + delete +} + +func newDeleteRequest(cfg *common.Config) (transform, error) { + delete, err := newDelete(cfg) + if err != nil { + return nil, err + } + + switch delete.targetInfo.Type { + case targetBody: + delete.run = deleteBody + case targetHeader: + delete.run = deleteHeader + case targetURLParams: + delete.run = deleteURLParams + default: + return nil, fmt.Errorf("invalid target type: %s", delete.targetInfo.Type) + } + + return &deleteRequest{delete: delete}, nil +} + +func (deleteReq *deleteRequest) run(ctx transformContext, req *request) (*request, error) { + transformable := &transformable{ + body: req.body, + header: req.header, + url: req.url, + } + if err := deleteReq.delete.runDelete(ctx, transformable); err != nil { + return nil, err + } + return req, nil +} + +func newDeleteResponse(cfg *common.Config) (transform, error) { + delete, err := newDelete(cfg) + if err != nil { + return nil, err + } + + switch delete.targetInfo.Type { + case targetBody: + delete.run = deleteBody + default: + return nil, fmt.Errorf("invalid target type: %s", delete.targetInfo.Type) + } + + return &deleteResponse{delete: delete}, nil +} + +func (deleteRes *deleteResponse) run(ctx transformContext, res *response) (*response, error) { + transformable := &transformable{ + body: res.body, + header: res.header, + url: res.url, + } + if err := deleteRes.delete.runDelete(ctx, transformable); err != nil { + return nil, err + } + return res, nil +} + +func newDeletePagination(cfg *common.Config) (transform, error) { + delete, err := newDelete(cfg) + if err != nil { + return nil, err + } + + switch delete.targetInfo.Type { + case targetBody: + delete.run = deleteBody + case targetHeader: + delete.run = deleteHeader + case targetURLParams: + delete.run = deleteURLParams + default: + return nil, fmt.Errorf("invalid target type: %s", delete.targetInfo.Type) + } + + return &deletePagination{delete: delete}, nil +} + +func (deletePag *deletePagination) run(ctx transformContext, pag *pagination) (*pagination, error) { + transformable := &transformable{ + body: pag.body, + header: pag.header, + url: pag.url, + } + if err := deletePag.delete.runDelete(ctx, transformable); err != nil { + return nil, err + } + return pag, nil +} + +func newDelete(cfg *common.Config) (delete, error) { + c := &deleteConfig{} + if err := cfg.Unpack(c); err != nil { + return delete{}, errors.Wrap(err, "fail to unpack the delete configuration") + } + + ti, err := getTargetInfo(c.Target) + if err != nil { + return delete{}, err + } + + return delete{ + targetInfo: ti, + }, nil +} + +func (delete *delete) runDelete(ctx transformContext, transformable *transformable) error { + return delete.run(ctx, transformable, delete.targetInfo.Name) +} + +func deleteFromCommonMap(m common.MapStr, key string) error { + if err := m.Delete(key); err != common.ErrKeyNotFound { + return err + } + return nil +} + +func deleteBody(ctx transformContext, transformable *transformable, key string) error { + return deleteFromCommonMap(transformable.body, key) +} + +func deleteHeader(ctx transformContext, transformable *transformable, key string) error { + transformable.header.Del(key) + return nil +} + +func deleteURLParams(ctx transformContext, transformable *transformable, key string) error { + transformable.url.Query().Del(key) + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_set.go b/x-pack/filebeat/input/httpjson/v2/transfrom_set.go new file mode 100644 index 00000000000..d0fddd292c5 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_set.go @@ -0,0 +1,194 @@ +package v2 + +import ( + "fmt" + httpURL "net/url" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/pkg/errors" +) + +const setName = "set" + +var ( + _ requestTransform = &setRequest{} + _ responseTransform = &setResponse{} + _ paginationTransform = &setPagination{} +) + +type setConfig struct { + Target string `config:"target"` + Value *valueTpl `config:"value"` + Default string `config:"default"` +} + +type set struct { + targetInfo targetInfo + value *valueTpl + defaultValue string + + run func(ctx transformContext, transformable *transformable, key, val string) error +} + +func (set) transformName() string { return setName } + +type setRequest struct { + set +} + +type setResponse struct { + set +} + +type setPagination struct { + set +} + +func newSetRequest(cfg *common.Config) (transform, error) { + set, err := newSet(cfg) + if err != nil { + return nil, err + } + + switch set.targetInfo.Type { + case targetBody: + set.run = setBody + case targetHeader: + set.run = setHeader + case targetURLParams: + set.run = setURLParams + default: + return nil, fmt.Errorf("invalid target type: %s", set.targetInfo.Type) + } + + return &setRequest{set: set}, nil +} + +func (setReq *setRequest) run(ctx transformContext, req *request) (*request, error) { + transformable := &transformable{ + body: req.body, + header: req.header, + url: req.url, + } + if err := setReq.set.runSet(ctx, transformable); err != nil { + return nil, err + } + return req, nil +} + +func newSetResponse(cfg *common.Config) (transform, error) { + set, err := newSet(cfg) + if err != nil { + return nil, err + } + + switch set.targetInfo.Type { + case targetBody: + set.run = setBody + default: + return nil, fmt.Errorf("invalid target type: %s", set.targetInfo.Type) + } + + return &setResponse{set: set}, nil +} + +func (setRes *setResponse) run(ctx transformContext, res *response) (*response, error) { + transformable := &transformable{ + body: res.body, + header: res.header, + url: res.url, + } + if err := setRes.set.runSet(ctx, transformable); err != nil { + return nil, err + } + return res, nil +} + +func newSetPagination(cfg *common.Config) (transform, error) { + set, err := newSet(cfg) + if err != nil { + return nil, err + } + + switch set.targetInfo.Type { + case targetBody: + set.run = setBody + case targetHeader: + set.run = setHeader + case targetURLParams: + set.run = setURLParams + case targetURLValue: + set.run = setURLValue + default: + return nil, fmt.Errorf("invalid target type: %s", set.targetInfo.Type) + } + + return &setPagination{set: set}, nil +} + +func (setPag *setPagination) run(ctx transformContext, pag *pagination) (*pagination, error) { + transformable := &transformable{ + body: pag.body, + header: pag.header, + url: pag.url, + } + if err := setPag.set.runSet(ctx, transformable); err != nil { + return nil, err + } + return pag, nil +} + +func newSet(cfg *common.Config) (set, error) { + c := &setConfig{} + if err := cfg.Unpack(c); err != nil { + return set{}, errors.Wrap(err, "fail to unpack the set configuration") + } + + ti, err := getTargetInfo(c.Target) + if err != nil { + return set{}, err + } + + return set{ + targetInfo: ti, + value: c.Value, + defaultValue: c.Default, + }, nil +} + +func (set *set) runSet(ctx transformContext, transformable *transformable) error { + value := set.value.Execute(set.defaultValue) + return set.run(ctx, transformable, set.targetInfo.Name, value) +} + +func setToCommonMap(m common.MapStr, key, val string) error { + if _, err := m.Put(key, val); err != nil { + return err + } + return nil +} + +func setBody(ctx transformContext, transformable *transformable, key, value string) error { + return setToCommonMap(transformable.body, key, value) +} + +func setHeader(ctx transformContext, transformable *transformable, key, value string) error { + transformable.header.Add(key, value) + return nil +} + +func setURLParams(ctx transformContext, transformable *transformable, key, value string) error { + transformable.url.Query().Add(key, value) + return nil +} + +func setURLValue(ctx transformContext, transformable *transformable, _, value string) error { + query := transformable.url.Query().Encode() + url, err := httpURL.Parse(value) + if err != nil { + return err + } + url.RawQuery = query + transformable.url = url + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_split.go b/x-pack/filebeat/input/httpjson/v2/transfrom_split.go new file mode 100644 index 00000000000..5ec3cc8e833 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_split.go @@ -0,0 +1 @@ +package v2 diff --git a/x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go similarity index 52% rename from x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go rename to x-pack/filebeat/input/httpjson/v2/value_tpl.go index 552b71bda04..bf18c2ffeb3 100644 --- a/x-pack/filebeat/input/httpjson/v2/internal/transforms/template_funcs.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -1,6 +1,57 @@ -package transforms +package v2 -import "time" +import ( + "text/template" + "time" +) + +type valueTpl struct { + *template.Template +} + +func (t *valueTpl) Unpack(in string) error { + tpl, err := template.New(""). + Option("missingkey=error"). + Funcs(template.FuncMap{ + "now": now, + "formatDate": formatDate, + "parseDate": parseDate, + "getRFC5988Link": getRFC5988Link, + }). + Parse(in) + if err != nil { + return err + } + + *t = valueTpl{Template: tpl} + + return nil +} + +func (t *valueTpl) Execute(defaultVal string) (val string) { + // defer func() { + // if r := recover(); r != nil { + // // really ugly + // val = defaultVal + // } + // }() + + // buf := new(bytes.Buffer) + // data := map[string]interface{}{ + // "header": tr.Headers.Clone(), + // "body": tr.Body.Clone(), + // "url.value": tr.URL.String(), + // "url.params": tr.URL.Query(), + // // "cursor": tr.Cursor.Clone(), + // // "last_event": tr.LastEvent, + // // "last_response": tr.LastResponse.Clone(), + // } + // if err := t.Template.Execute(buf, data); err != nil { + // return defaultVal + // } + // return buf.String() + return "" +} var ( predefinedLayouts = map[string]string{ From 8b8e9d62f26e71599e08e4286418c0b7770edf79 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 29 Oct 2020 15:04:41 +0100 Subject: [PATCH 06/35] Run fmt --- x-pack/filebeat/input/httpjson/v2/config.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/config_auth.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/config_request.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/config_response.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/pagination.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/request.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/response.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/transform.go | 7 ++++++- x-pack/filebeat/input/httpjson/v2/transform_registry.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/transform_target.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/transfrom_append.go | 7 ++++++- x-pack/filebeat/input/httpjson/v2/transfrom_delete.go | 7 ++++++- x-pack/filebeat/input/httpjson/v2/transfrom_split.go | 4 ++++ x-pack/filebeat/input/httpjson/v2/value_tpl.go | 4 ++++ 14 files changed, 62 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/v2/config.go b/x-pack/filebeat/input/httpjson/v2/config.go index fec00aac041..da777d8c3bb 100644 --- a/x-pack/filebeat/input/httpjson/v2/config.go +++ b/x-pack/filebeat/input/httpjson/v2/config.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/config_auth.go b/x-pack/filebeat/input/httpjson/v2/config_auth.go index 5dd90c122f7..ed7a23e9caf 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_auth.go +++ b/x-pack/filebeat/input/httpjson/v2/config_auth.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/config_request.go b/x-pack/filebeat/input/httpjson/v2/config_request.go index 53e55299ba7..53c989fe842 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/v2/config_request.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/config_response.go b/x-pack/filebeat/input/httpjson/v2/config_response.go index 86c5d22d621..978457ea40a 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/v2/config_response.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 type responseConfig struct { diff --git a/x-pack/filebeat/input/httpjson/v2/pagination.go b/x-pack/filebeat/input/httpjson/v2/pagination.go index afdd4f66e18..2271b8c1091 100644 --- a/x-pack/filebeat/input/httpjson/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/v2/pagination.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/v2/request.go index a636261c27f..e409a3272e6 100644 --- a/x-pack/filebeat/input/httpjson/v2/request.go +++ b/x-pack/filebeat/input/httpjson/v2/request.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index 6cab31c8e43..f61ea89a870 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go index 6e630339b66..6aa85b62b2a 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( @@ -6,9 +10,10 @@ import ( "net/url" "strings" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/pkg/errors" ) const logName = "httpjson.transforms" diff --git a/x-pack/filebeat/input/httpjson/v2/transform_registry.go b/x-pack/filebeat/input/httpjson/v2/transform_registry.go index d75e7faca0e..e585fc42e04 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform_registry.go +++ b/x-pack/filebeat/input/httpjson/v2/transform_registry.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/transform_target.go b/x-pack/filebeat/input/httpjson/v2/transform_target.go index 2741284c8fd..630976aae9a 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform_target.go +++ b/x-pack/filebeat/input/httpjson/v2/transform_target.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go index 20243c00f92..6bd8da233c5 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go @@ -1,10 +1,15 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" ) const appendName = "append" diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go b/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go index 4a64556d228..be4394df886 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go @@ -1,10 +1,15 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" ) const deleteName = "delete" diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_split.go b/x-pack/filebeat/input/httpjson/v2/transfrom_split.go index 5ec3cc8e833..d4a74a89e77 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_split.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_split.go @@ -1 +1,5 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index bf18c2ffeb3..e2cb038c4aa 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( From de80d3fa46ab5b95d48677cf03fd4c66445ca74f Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 29 Oct 2020 16:11:41 +0100 Subject: [PATCH 07/35] Set tests and fix tpl --- .../input/httpjson/v2/transform_set_test.go | 88 +++++++++++++++++++ .../input/httpjson/v2/transfrom_append.go | 2 +- .../input/httpjson/v2/transfrom_set.go | 6 +- .../filebeat/input/httpjson/v2/value_tpl.go | 47 +++++----- 4 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/v2/transform_set_test.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/v2/transform_set_test.go new file mode 100644 index 00000000000..8f7bd6f8bf0 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transform_set_test.go @@ -0,0 +1,88 @@ +package v2 + +import ( + "net/http" + "net/url" + "testing" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func TestSetFunctions(t *testing.T) { + cases := []struct { + name string + tfunc func(ctx transformContext, transformable *transformable, key, val string) error + paramCtx transformContext + paramTr *transformable + paramKey string + paramVal string + expectedTr *transformable + expectedErr error + }{ + { + name: "setBody", + tfunc: setBody, + paramCtx: transformContext{}, + paramTr: &transformable{body: common.MapStr{}}, + paramKey: "a_key", + paramVal: "a_value", + expectedTr: &transformable{body: common.MapStr{"a_key": "a_value"}}, + expectedErr: nil, + }, + { + name: "setHeader", + tfunc: setHeader, + paramCtx: transformContext{}, + paramTr: &transformable{header: http.Header{}}, + paramKey: "a_key", + paramVal: "a_value", + expectedTr: &transformable{header: http.Header{"A_key": []string{"a_value"}}}, + expectedErr: nil, + }, + { + name: "setURLParams", + tfunc: setURLParams, + paramCtx: transformContext{}, + paramTr: &transformable{url: func() *url.URL { + u, _ := url.Parse("http://foo.example.com") + return u + }()}, + paramKey: "a_key", + paramVal: "a_value", + expectedTr: &transformable{url: func() *url.URL { + u, _ := url.Parse("http://foo.example.com?a_key=a_value") + return u + }()}, + expectedErr: nil, + }, + { + name: "setURLValue", + tfunc: setURLValue, + paramCtx: transformContext{}, + paramTr: &transformable{url: func() *url.URL { + u, _ := url.Parse("http://foo.example.com") + return u + }()}, + paramVal: "http://different.example.com", + expectedTr: &transformable{url: func() *url.URL { + u, _ := url.Parse("http://different.example.com") + return u + }()}, + expectedErr: nil, + }, + } + + for _, tcase := range cases { + tcase := tcase + t.Run(tcase.name, func(t *testing.T) { + gotErr := tcase.tfunc(tcase.paramCtx, tcase.paramTr, tcase.paramKey, tcase.paramVal) + if tcase.expectedErr == nil { + assert.NoError(t, gotErr) + } else { + assert.EqualError(t, gotErr, tcase.expectedErr.Error()) + } + assert.EqualValues(t, tcase.expectedTr, tcase.paramTr) + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go index 6bd8da233c5..370c2deb8db 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go @@ -159,7 +159,7 @@ func newAppend(cfg *common.Config) (appendt, error) { } func (append *appendt) runAppend(ctx transformContext, transformable *transformable) error { - value := append.value.Execute(append.defaultValue) + value := append.value.Execute(ctx, transformable, append.defaultValue) return append.run(ctx, transformable, append.targetInfo.Name, value) } diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_set.go b/x-pack/filebeat/input/httpjson/v2/transfrom_set.go index d0fddd292c5..36c2ad11818 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_set.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_set.go @@ -157,7 +157,7 @@ func newSet(cfg *common.Config) (set, error) { } func (set *set) runSet(ctx transformContext, transformable *transformable) error { - value := set.value.Execute(set.defaultValue) + value := set.value.Execute(ctx, transformable, set.defaultValue) return set.run(ctx, transformable, set.targetInfo.Name, value) } @@ -178,7 +178,9 @@ func setHeader(ctx transformContext, transformable *transformable, key, value st } func setURLParams(ctx transformContext, transformable *transformable, key, value string) error { - transformable.url.Query().Add(key, value) + q := transformable.url.Query() + q.Add(key, value) + transformable.url.RawQuery = q.Encode() return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index e2cb038c4aa..9e5d09d56b0 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -5,6 +5,7 @@ package v2 import ( + "bytes" "text/template" "time" ) @@ -32,29 +33,29 @@ func (t *valueTpl) Unpack(in string) error { return nil } -func (t *valueTpl) Execute(defaultVal string) (val string) { - // defer func() { - // if r := recover(); r != nil { - // // really ugly - // val = defaultVal - // } - // }() - - // buf := new(bytes.Buffer) - // data := map[string]interface{}{ - // "header": tr.Headers.Clone(), - // "body": tr.Body.Clone(), - // "url.value": tr.URL.String(), - // "url.params": tr.URL.Query(), - // // "cursor": tr.Cursor.Clone(), - // // "last_event": tr.LastEvent, - // // "last_response": tr.LastResponse.Clone(), - // } - // if err := t.Template.Execute(buf, data); err != nil { - // return defaultVal - // } - // return buf.String() - return "" +func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal string) (val string) { + defer func() { + if r := recover(); r != nil { + // TODO: find alternative to this ugliness + val = defaultVal + } + }() + + buf := new(bytes.Buffer) + data := map[string]interface{}{ + "header": tr.header.Clone(), + "body": tr.body.Clone(), + "url.value": tr.url.String(), + "url.params": tr.url.Query(), + "cursor": trCtx.cursor.Clone(), + "last_event": trCtx.lastEvent.Clone(), + "last_response": trCtx.lastResponse.Clone(), + } + if err := t.Template.Execute(buf, data); err != nil { + return defaultVal + } + + return buf.String() } var ( From 43d241efad19036124ac78211483145e310bcb1a Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 30 Oct 2020 09:04:19 +0100 Subject: [PATCH 08/35] basic processing structure --- x-pack/filebeat/input/httpjson/v2/input.go | 3 +- x-pack/filebeat/input/httpjson/v2/request.go | 42 +++++++++++++------ x-pack/filebeat/input/httpjson/v2/response.go | 40 ++++++++++++++++++ .../filebeat/input/httpjson/v2/transform.go | 11 ++++- 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go index 51dcdbb02d3..52fac3f2f2c 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -132,7 +132,8 @@ func run( } requestFactory := newRequestFactory(config.Request, config.Auth, log) - requester := newRequester(httpClient, requestFactory, log) + responseProcessor := &responseProcessor{} + requester := newRequester(httpClient, requestFactory, responseProcessor, log) // loadContextFromCursor trCtx := transformContext{} diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/v2/request.go index e409a3272e6..f7961d2eefc 100644 --- a/x-pack/filebeat/input/httpjson/v2/request.go +++ b/x-pack/filebeat/input/httpjson/v2/request.go @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "net/http" "net/url" @@ -126,16 +125,22 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transform } type requester struct { - log *logp.Logger - client *http.Client - requestFactory *requestFactory + log *logp.Logger + client *http.Client + requestFactory *requestFactory + responseProcessor *responseProcessor } -func newRequester(client *http.Client, requestFactory *requestFactory, log *logp.Logger) *requester { +func newRequester( + client *http.Client, + requestFactory *requestFactory, + responseProcessor *responseProcessor, + log *logp.Logger) *requester { return &requester{ - log: log, - client: client, - requestFactory: requestFactory, + log: log, + client: client, + requestFactory: requestFactory, + responseProcessor: responseProcessor, } } @@ -145,13 +150,26 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu return fmt.Errorf("failed to create http request: %w", err) } - resp, err := r.client.Do(req) + httpResp, err := r.client.Do(req) if err != nil { return fmt.Errorf("failed to execute http client.Do: %w", err) } - body, _ := ioutil.ReadAll(resp.Body) - resp.Body.Close() + defer httpResp.Body.Close() + + eventsCh, err := r.responseProcessor.startProcessing(stdCtx, trCtx) + if err != nil { + return err + } + + for maybeEvent := range eventsCh { + if maybeEvent.failed() { + r.log.Errorf("error processing response: %v", maybeEvent) + continue + } + if err := publisher.Publish(maybeEvent.event, trCtx.cursor.Clone()); err != nil { + r.log.Errorf("error publishing event: %v", err) + } + } - fmt.Println(string(body)) return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index f61ea89a870..5df2becaec6 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -5,6 +5,7 @@ package v2 import ( + "context" "net/http" "net/url" @@ -24,3 +25,42 @@ type response struct { header http.Header url *url.URL } + +type responseProcessor struct { + splitTransform splitTransform + transforms []responseTransform + pagination *pagination +} + +func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx transformContext, resp *http.Response) (<-chan maybeEvent, error) { + ch := make(chan maybeEvent) + + go func() { + defer close(ch) + + iter := rp.pagination.newPageIterator(stdCtx, trCtx, resp) + for iter.hasNext() { + var err error + page := iter.page() + + for _, t := range rp.transforms { + page, err = t.run(trCtx, page) + if err != nil { + ch <- maybeEvent{err: err} + return + } + } + + if rp.splitTransform == nil { + continue + } + + if err := rp.splitTransform.run(trCtx, page, ch); err != nil { + ch <- maybeEvent{err: err} + return + } + } + }() + + return ch, nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go index 6aa85b62b2a..404e110ab0b 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -53,9 +53,18 @@ type paginationTransform interface { run(transformContext, *pagination) (*pagination, error) } +type maybeEvent struct { + err error + event beat.Event +} + +func (e maybeEvent) failed() bool { return e.err != nil } + +func (e maybeEvent) Error() string { return e.err.Error() } + type splitTransform interface { transform - run(transformContext, *response, []beat.Event) ([]beat.Event, error) + run(transformContext, *response, <-chan maybeEvent) error } type cursorTransform interface { From 5c6092ea32e00abe17a1616adb7ba88d4d56df14 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 30 Oct 2020 12:24:04 +0100 Subject: [PATCH 09/35] Add pagination mechanism --- .../input/httpjson/v2/config_request.go | 2 +- .../input/httpjson/v2/config_response.go | 5 +- x-pack/filebeat/input/httpjson/v2/input.go | 7 +- .../filebeat/input/httpjson/v2/pagination.go | 127 +++++++++++++++++- x-pack/filebeat/input/httpjson/v2/request.go | 27 +--- x-pack/filebeat/input/httpjson/v2/response.go | 36 +++-- .../filebeat/input/httpjson/v2/transform.go | 81 ++++------- .../input/httpjson/v2/transform_set_test.go | 43 +++--- .../input/httpjson/v2/transfrom_append.go | 83 +++--------- .../input/httpjson/v2/transfrom_delete.go | 83 +++--------- .../input/httpjson/v2/transfrom_set.go | 87 +++--------- .../filebeat/input/httpjson/v2/value_tpl.go | 28 ++-- .../input/httpjson/v2/value_tpl_test.go | 65 +++++++++ 13 files changed, 337 insertions(+), 337 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/v2/value_tpl_test.go diff --git a/x-pack/filebeat/input/httpjson/v2/config_request.go b/x-pack/filebeat/input/httpjson/v2/config_request.go index 53c989fe842..95d2a7bc27e 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/v2/config_request.go @@ -117,7 +117,7 @@ func (c *requestConfig) Validate() error { return errors.New("timeout must be greater than 0") } - if _, err := newRequestTransformsFromConfig(c.Transforms); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, requestNamespace); err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/v2/config_response.go b/x-pack/filebeat/input/httpjson/v2/config_response.go index 978457ea40a..e42f8c8b878 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/v2/config_response.go @@ -10,12 +10,11 @@ type responseConfig struct { } func (c *responseConfig) Validate() error { - if _, err := newResponseTransformsFromConfig(c.Transforms); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace); err != nil { return err } - if _, err := newPaginationTransformsFromConfig(c.Transforms); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, paginationNamespace); err != nil { return err } - return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go index 52fac3f2f2c..9923540795e 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -132,11 +132,14 @@ func run( } requestFactory := newRequestFactory(config.Request, config.Auth, log) - responseProcessor := &responseProcessor{} + pagination := newPagination(config, httpClient, log) + responseProcessor := newResponseProcessor(config.Response, pagination) requester := newRequester(httpClient, requestFactory, responseProcessor, log) // loadContextFromCursor - trCtx := transformContext{} + trCtx := emptyTransformContext() + // + err = timed.Periodic(stdCtx, config.Interval, func() error { log.Info("Process another repeated request.") diff --git a/x-pack/filebeat/input/httpjson/v2/pagination.go b/x-pack/filebeat/input/httpjson/v2/pagination.go index 2271b8c1091..ea4039fb35a 100644 --- a/x-pack/filebeat/input/httpjson/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/v2/pagination.go @@ -5,10 +5,14 @@ package v2 import ( + "context" + "encoding/json" + "io/ioutil" "net/http" "net/url" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) const paginationNamespace = "pagination" @@ -20,11 +24,124 @@ func registerPaginationTransforms() { } type pagination struct { - body common.MapStr - header http.Header - url *url.URL + httpClient *http.Client + requestFactory *requestFactory } -func (p *pagination) nextPageRequest() (*http.Request, error) { - return nil, nil +func newPagination(config config, httpClient *http.Client, log *logp.Logger) *pagination { + pagination := &pagination{httpClient: httpClient} + if config.Response == nil { + return pagination + } + ts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace) + requestFactory := newPaginationRequestFactory( + config.Request.Method, + *config.Request.URL.URL, + ts, + config.Auth, + log, + ) + pagination.requestFactory = requestFactory + return pagination +} + +func newPaginationRequestFactory(method string, url url.URL, ts []basicTransform, authConfig *authConfig, log *logp.Logger) *requestFactory { + // config validation already checked for errors here + rf := &requestFactory{ + url: url, + method: method, + body: &common.MapStr{}, + transforms: ts, + log: log, + } + if authConfig != nil && authConfig.Basic.isEnabled() { + rf.user = authConfig.Basic.User + rf.password = authConfig.Basic.Password + } + return rf +} + +type pageIterator struct { + pagination *pagination + + stdCtx context.Context + trCtx transformContext + + resp *http.Response + + isFirst bool +} + +func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformContext, resp *http.Response) *pageIterator { + return &pageIterator{ + pagination: p, + stdCtx: stdCtx, + trCtx: trCtx, + resp: resp, + isFirst: true, + } +} + +func (iter *pageIterator) next() (*transformable, bool, error) { + if iter == nil || iter.resp == nil { + return nil, false, nil + } + + if iter.isFirst { + iter.isFirst = false + tr, err := iter.getPage() + if err != nil { + return nil, false, err + } + return tr, true, nil + } + + httpReq, err := iter.pagination.requestFactory.newHTTPRequest(iter.stdCtx, iter.trCtx) + if err != nil { + return nil, false, err + } + + resp, err := iter.pagination.httpClient.Do(httpReq) + if err != nil { + return nil, false, err + } + + iter.resp = resp + + tr, err := iter.getPage() + if err != nil { + return nil, false, err + } + + if len(tr.body) == 0 { + return nil, false, nil + } + + return tr, true, nil +} + +func (iter *pageIterator) getPage() (*transformable, error) { + bodyBytes, err := ioutil.ReadAll(iter.resp.Body) + if err != nil { + return nil, err + } + iter.resp.Body.Close() + + tr := emptyTransformable() + tr.header = iter.resp.Header + tr.url = *iter.resp.Request.URL + + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, &tr.body); err != nil { + return nil, err + } + } + + iter.trCtx.lastResponse = &transformable{ + body: tr.body.Clone(), + header: tr.header.Clone(), + url: tr.url, + } + + return tr, nil } diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/v2/request.go index f7961d2eefc..b2fd3e3da22 100644 --- a/x-pack/filebeat/input/httpjson/v2/request.go +++ b/x-pack/filebeat/input/httpjson/v2/request.go @@ -25,28 +25,15 @@ func registerRequestTransforms() { registerTransform(requestNamespace, setName, newSetRequest) } -type request struct { - body common.MapStr - header http.Header - url *url.URL -} - -func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []requestTransform) (*request, error) { - req := &request{ - body: common.MapStr{}, - header: http.Header{}, - } - - clonedURL, err := url.Parse(url.String()) - if err != nil { - return nil, err - } - req.url = clonedURL +func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []basicTransform) (*transformable, error) { + req := emptyTransformable() + req.url = url if body != nil { req.body.DeepUpdate(*body) } + var err error for _, t := range trs { req, err = t.run(ctx, req) if err != nil { @@ -61,7 +48,7 @@ type requestFactory struct { url url.URL method string body *common.MapStr - transforms []requestTransform + transforms []basicTransform user string password string log *logp.Logger @@ -69,7 +56,7 @@ type requestFactory struct { func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { // config validation already checked for errors here - ts, _ := newRequestTransformsFromConfig(config.Transforms) + ts, _ := newBasicTransformsFromConfig(config.Transforms, requestNamespace) rf := &requestFactory{ url: *config.URL.URL, method: config.Method, @@ -156,7 +143,7 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu } defer httpResp.Body.Close() - eventsCh, err := r.responseProcessor.startProcessing(stdCtx, trCtx) + eventsCh, err := r.responseProcessor.startProcessing(stdCtx, trCtx, httpResp) if err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index 5df2becaec6..b8c6c4c1de3 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -7,9 +7,6 @@ package v2 import ( "context" "net/http" - "net/url" - - "github.com/elastic/beats/v7/libbeat/common" ) const responseNamespace = "response" @@ -20,18 +17,24 @@ func registerResponseTransforms() { registerTransform(responseNamespace, setName, newSetResponse) } -type response struct { - body common.MapStr - header http.Header - url *url.URL -} - type responseProcessor struct { splitTransform splitTransform - transforms []responseTransform + transforms []basicTransform pagination *pagination } +func newResponseProcessor(config *responseConfig, pagination *pagination) *responseProcessor { + rp := &responseProcessor{ + pagination: pagination, + } + if config == nil { + return rp + } + ts, _ := newBasicTransformsFromConfig(config.Transforms, responseNamespace) + rp.transforms = ts + return rp +} + func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx transformContext, resp *http.Response) (<-chan maybeEvent, error) { ch := make(chan maybeEvent) @@ -39,9 +42,16 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans defer close(ch) iter := rp.pagination.newPageIterator(stdCtx, trCtx, resp) - for iter.hasNext() { - var err error - page := iter.page() + for { + page, hasNext, err := iter.next() + if err != nil { + ch <- maybeEvent{err: err} + return + } + + if !hasNext { + return + } for _, t := range rp.transforms { page, err = t.run(trCtx, page) diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go index 404e110ab0b..0ba0d5263b4 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -25,32 +25,37 @@ type transforms []transform type transformContext struct { cursor common.MapStr lastEvent common.MapStr - lastResponse common.MapStr + lastResponse *transformable +} + +func emptyTransformContext() transformContext { + return transformContext{ + cursor: make(common.MapStr), + lastEvent: make(common.MapStr), + lastResponse: emptyTransformable(), + } } type transformable struct { body common.MapStr header http.Header - url *url.URL -} - -type transform interface { - transformName() string + url url.URL } -type requestTransform interface { - transform - run(transformContext, *request) (*request, error) +func emptyTransformable() *transformable { + return &transformable{ + body: make(common.MapStr), + header: make(http.Header), + } } -type responseTransform interface { - transform - run(transformContext, *response) (*response, error) +type transform interface { + transformName() string } -type paginationTransform interface { +type basicTransform interface { transform - run(transformContext, *pagination) (*pagination, error) + run(transformContext, *transformable) (*transformable, error) } type maybeEvent struct { @@ -64,7 +69,7 @@ func (e maybeEvent) Error() string { return e.err.Error() } type splitTransform interface { transform - run(transformContext, *response, <-chan maybeEvent) error + run(transformContext, *transformable, <-chan maybeEvent) error } type cursorTransform interface { @@ -108,53 +113,17 @@ func newTransformsFromConfig(config transformsConfig, namespace string) (transfo return trans, nil } -func newRequestTransformsFromConfig(config transformsConfig) ([]requestTransform, error) { - ts, err := newTransformsFromConfig(config, requestNamespace) - if err != nil { - return nil, err - } - - var rts []requestTransform - for _, t := range ts { - rt, ok := t.(requestTransform) - if !ok { - return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), requestNamespace) - } - rts = append(rts, rt) - } - - return rts, nil -} - -func newResponseTransformsFromConfig(config transformsConfig) ([]responseTransform, error) { - ts, err := newTransformsFromConfig(config, responseNamespace) - if err != nil { - return nil, err - } - - var rts []responseTransform - for _, t := range ts { - rt, ok := t.(responseTransform) - if !ok { - return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), responseNamespace) - } - rts = append(rts, rt) - } - - return rts, nil -} - -func newPaginationTransformsFromConfig(config transformsConfig) ([]paginationTransform, error) { - ts, err := newTransformsFromConfig(config, paginationNamespace) +func newBasicTransformsFromConfig(config transformsConfig, namespace string) ([]basicTransform, error) { + ts, err := newTransformsFromConfig(config, namespace) if err != nil { return nil, err } - var rts []paginationTransform + var rts []basicTransform for _, t := range ts { - rt, ok := t.(paginationTransform) + rt, ok := t.(basicTransform) if !ok { - return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), paginationNamespace) + return nil, fmt.Errorf("transform %s is not a valid %s transform", t.transformName(), namespace) } rts = append(rts, rt) } diff --git a/x-pack/filebeat/input/httpjson/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/v2/transform_set_test.go index 8f7bd6f8bf0..c94f9df5250 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform_set_test.go +++ b/x-pack/filebeat/input/httpjson/v2/transform_set_test.go @@ -41,34 +41,22 @@ func TestSetFunctions(t *testing.T) { expectedErr: nil, }, { - name: "setURLParams", - tfunc: setURLParams, - paramCtx: transformContext{}, - paramTr: &transformable{url: func() *url.URL { - u, _ := url.Parse("http://foo.example.com") - return u - }()}, - paramKey: "a_key", - paramVal: "a_value", - expectedTr: &transformable{url: func() *url.URL { - u, _ := url.Parse("http://foo.example.com?a_key=a_value") - return u - }()}, + name: "setURLParams", + tfunc: setURLParams, + paramCtx: transformContext{}, + paramTr: &transformable{url: newURL("http://foo.example.com")}, + paramKey: "a_key", + paramVal: "a_value", + expectedTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value")}, expectedErr: nil, }, { - name: "setURLValue", - tfunc: setURLValue, - paramCtx: transformContext{}, - paramTr: &transformable{url: func() *url.URL { - u, _ := url.Parse("http://foo.example.com") - return u - }()}, - paramVal: "http://different.example.com", - expectedTr: &transformable{url: func() *url.URL { - u, _ := url.Parse("http://different.example.com") - return u - }()}, + name: "setURLValue", + tfunc: setURLValue, + paramCtx: transformContext{}, + paramTr: &transformable{url: newURL("http://foo.example.com")}, + paramVal: "http://different.example.com", + expectedTr: &transformable{url: newURL("http://different.example.com")}, expectedErr: nil, }, } @@ -86,3 +74,8 @@ func TestSetFunctions(t *testing.T) { }) } } + +func newURL(u string) url.URL { + url, _ := url.Parse(u) + return *url +} diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go index 370c2deb8db..650557b2d1c 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_append.go @@ -14,12 +14,6 @@ import ( const appendName = "append" -var ( - _ requestTransform = &appendRequest{} - _ responseTransform = &appendResponse{} - _ paginationTransform = &appendPagination{} -) - type appendConfig struct { Target string `config:"target"` Value *valueTpl `config:"value"` @@ -31,23 +25,11 @@ type appendt struct { value *valueTpl defaultValue string - run func(ctx transformContext, transformable *transformable, key, val string) error + runFunc func(ctx transformContext, transformable *transformable, key, val string) error } func (appendt) transformName() string { return appendName } -type appendRequest struct { - appendt -} - -type appendResponse struct { - appendt -} - -type appendPagination struct { - appendt -} - func newAppendRequest(cfg *common.Config) (transform, error) { append, err := newAppend(cfg) if err != nil { @@ -56,28 +38,16 @@ func newAppendRequest(cfg *common.Config) (transform, error) { switch append.targetInfo.Type { case targetBody: - append.run = appendBody + append.runFunc = appendBody case targetHeader: - append.run = appendHeader + append.runFunc = appendHeader case targetURLParams: - append.run = appendURLParams + append.runFunc = appendURLParams default: return nil, fmt.Errorf("invalid target type: %s", append.targetInfo.Type) } - return &appendRequest{appendt: append}, nil -} - -func (appendReq *appendRequest) run(ctx transformContext, req *request) (*request, error) { - transformable := &transformable{ - body: req.body, - header: req.header, - url: req.url, - } - if err := appendReq.appendt.runAppend(ctx, transformable); err != nil { - return nil, err - } - return req, nil + return &append, nil } func newAppendResponse(cfg *common.Config) (transform, error) { @@ -88,24 +58,12 @@ func newAppendResponse(cfg *common.Config) (transform, error) { switch append.targetInfo.Type { case targetBody: - append.run = appendBody + append.runFunc = appendBody default: return nil, fmt.Errorf("invalid target type: %s", append.targetInfo.Type) } - return &appendResponse{appendt: append}, nil -} - -func (appendRes *appendResponse) run(ctx transformContext, res *response) (*response, error) { - transformable := &transformable{ - body: res.body, - header: res.header, - url: res.url, - } - if err := appendRes.appendt.runAppend(ctx, transformable); err != nil { - return nil, err - } - return res, nil + return &append, nil } func newAppendPagination(cfg *common.Config) (transform, error) { @@ -116,28 +74,16 @@ func newAppendPagination(cfg *common.Config) (transform, error) { switch append.targetInfo.Type { case targetBody: - append.run = appendBody + append.runFunc = appendBody case targetHeader: - append.run = appendHeader + append.runFunc = appendHeader case targetURLParams: - append.run = appendURLParams + append.runFunc = appendURLParams default: return nil, fmt.Errorf("invalid target type: %s", append.targetInfo.Type) } - return &appendPagination{appendt: append}, nil -} - -func (appendPag *appendPagination) run(ctx transformContext, pag *pagination) (*pagination, error) { - transformable := &transformable{ - body: pag.body, - header: pag.header, - url: pag.url, - } - if err := appendPag.appendt.runAppend(ctx, transformable); err != nil { - return nil, err - } - return pag, nil + return &append, nil } func newAppend(cfg *common.Config) (appendt, error) { @@ -158,9 +104,12 @@ func newAppend(cfg *common.Config) (appendt, error) { }, nil } -func (append *appendt) runAppend(ctx transformContext, transformable *transformable) error { +func (append *appendt) run(ctx transformContext, transformable *transformable) (*transformable, error) { value := append.value.Execute(ctx, transformable, append.defaultValue) - return append.run(ctx, transformable, append.targetInfo.Name, value) + if err := append.runFunc(ctx, transformable, append.targetInfo.Name, value); err != nil { + return nil, err + } + return transformable, nil } func appendToCommonMap(m common.MapStr, key, val string) error { diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go b/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go index be4394df886..085bbb7a5a8 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go @@ -14,12 +14,6 @@ import ( const deleteName = "delete" -var ( - _ requestTransform = &deleteRequest{} - _ responseTransform = &deleteResponse{} - _ paginationTransform = &deletePagination{} -) - type deleteConfig struct { Target string `config:"target"` } @@ -27,23 +21,11 @@ type deleteConfig struct { type delete struct { targetInfo targetInfo - run func(ctx transformContext, transformable *transformable, key string) error + runFunc func(ctx transformContext, transformable *transformable, key string) error } func (delete) transformName() string { return deleteName } -type deleteRequest struct { - delete -} - -type deleteResponse struct { - delete -} - -type deletePagination struct { - delete -} - func newDeleteRequest(cfg *common.Config) (transform, error) { delete, err := newDelete(cfg) if err != nil { @@ -52,28 +34,16 @@ func newDeleteRequest(cfg *common.Config) (transform, error) { switch delete.targetInfo.Type { case targetBody: - delete.run = deleteBody + delete.runFunc = deleteBody case targetHeader: - delete.run = deleteHeader + delete.runFunc = deleteHeader case targetURLParams: - delete.run = deleteURLParams + delete.runFunc = deleteURLParams default: return nil, fmt.Errorf("invalid target type: %s", delete.targetInfo.Type) } - return &deleteRequest{delete: delete}, nil -} - -func (deleteReq *deleteRequest) run(ctx transformContext, req *request) (*request, error) { - transformable := &transformable{ - body: req.body, - header: req.header, - url: req.url, - } - if err := deleteReq.delete.runDelete(ctx, transformable); err != nil { - return nil, err - } - return req, nil + return &delete, nil } func newDeleteResponse(cfg *common.Config) (transform, error) { @@ -84,24 +54,12 @@ func newDeleteResponse(cfg *common.Config) (transform, error) { switch delete.targetInfo.Type { case targetBody: - delete.run = deleteBody + delete.runFunc = deleteBody default: return nil, fmt.Errorf("invalid target type: %s", delete.targetInfo.Type) } - return &deleteResponse{delete: delete}, nil -} - -func (deleteRes *deleteResponse) run(ctx transformContext, res *response) (*response, error) { - transformable := &transformable{ - body: res.body, - header: res.header, - url: res.url, - } - if err := deleteRes.delete.runDelete(ctx, transformable); err != nil { - return nil, err - } - return res, nil + return &delete, nil } func newDeletePagination(cfg *common.Config) (transform, error) { @@ -112,28 +70,16 @@ func newDeletePagination(cfg *common.Config) (transform, error) { switch delete.targetInfo.Type { case targetBody: - delete.run = deleteBody + delete.runFunc = deleteBody case targetHeader: - delete.run = deleteHeader + delete.runFunc = deleteHeader case targetURLParams: - delete.run = deleteURLParams + delete.runFunc = deleteURLParams default: return nil, fmt.Errorf("invalid target type: %s", delete.targetInfo.Type) } - return &deletePagination{delete: delete}, nil -} - -func (deletePag *deletePagination) run(ctx transformContext, pag *pagination) (*pagination, error) { - transformable := &transformable{ - body: pag.body, - header: pag.header, - url: pag.url, - } - if err := deletePag.delete.runDelete(ctx, transformable); err != nil { - return nil, err - } - return pag, nil + return &delete, nil } func newDelete(cfg *common.Config) (delete, error) { @@ -152,8 +98,11 @@ func newDelete(cfg *common.Config) (delete, error) { }, nil } -func (delete *delete) runDelete(ctx transformContext, transformable *transformable) error { - return delete.run(ctx, transformable, delete.targetInfo.Name) +func (delete *delete) run(ctx transformContext, transformable *transformable) (*transformable, error) { + if err := delete.runFunc(ctx, transformable, delete.targetInfo.Name); err != nil { + return nil, err + } + return transformable, nil } func deleteFromCommonMap(m common.MapStr, key string) error { diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_set.go b/x-pack/filebeat/input/httpjson/v2/transfrom_set.go index 36c2ad11818..79ab7e438c4 100644 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_set.go +++ b/x-pack/filebeat/input/httpjson/v2/transfrom_set.go @@ -10,12 +10,6 @@ import ( const setName = "set" -var ( - _ requestTransform = &setRequest{} - _ responseTransform = &setResponse{} - _ paginationTransform = &setPagination{} -) - type setConfig struct { Target string `config:"target"` Value *valueTpl `config:"value"` @@ -27,23 +21,11 @@ type set struct { value *valueTpl defaultValue string - run func(ctx transformContext, transformable *transformable, key, val string) error + runFunc func(ctx transformContext, transformable *transformable, key, val string) error } func (set) transformName() string { return setName } -type setRequest struct { - set -} - -type setResponse struct { - set -} - -type setPagination struct { - set -} - func newSetRequest(cfg *common.Config) (transform, error) { set, err := newSet(cfg) if err != nil { @@ -52,28 +34,16 @@ func newSetRequest(cfg *common.Config) (transform, error) { switch set.targetInfo.Type { case targetBody: - set.run = setBody + set.runFunc = setBody case targetHeader: - set.run = setHeader + set.runFunc = setHeader case targetURLParams: - set.run = setURLParams + set.runFunc = setURLParams default: return nil, fmt.Errorf("invalid target type: %s", set.targetInfo.Type) } - return &setRequest{set: set}, nil -} - -func (setReq *setRequest) run(ctx transformContext, req *request) (*request, error) { - transformable := &transformable{ - body: req.body, - header: req.header, - url: req.url, - } - if err := setReq.set.runSet(ctx, transformable); err != nil { - return nil, err - } - return req, nil + return &set, nil } func newSetResponse(cfg *common.Config) (transform, error) { @@ -84,24 +54,12 @@ func newSetResponse(cfg *common.Config) (transform, error) { switch set.targetInfo.Type { case targetBody: - set.run = setBody + set.runFunc = setBody default: return nil, fmt.Errorf("invalid target type: %s", set.targetInfo.Type) } - return &setResponse{set: set}, nil -} - -func (setRes *setResponse) run(ctx transformContext, res *response) (*response, error) { - transformable := &transformable{ - body: res.body, - header: res.header, - url: res.url, - } - if err := setRes.set.runSet(ctx, transformable); err != nil { - return nil, err - } - return res, nil + return &set, nil } func newSetPagination(cfg *common.Config) (transform, error) { @@ -112,30 +70,18 @@ func newSetPagination(cfg *common.Config) (transform, error) { switch set.targetInfo.Type { case targetBody: - set.run = setBody + set.runFunc = setBody case targetHeader: - set.run = setHeader + set.runFunc = setHeader case targetURLParams: - set.run = setURLParams + set.runFunc = setURLParams case targetURLValue: - set.run = setURLValue + set.runFunc = setURLValue default: return nil, fmt.Errorf("invalid target type: %s", set.targetInfo.Type) } - return &setPagination{set: set}, nil -} - -func (setPag *setPagination) run(ctx transformContext, pag *pagination) (*pagination, error) { - transformable := &transformable{ - body: pag.body, - header: pag.header, - url: pag.url, - } - if err := setPag.set.runSet(ctx, transformable); err != nil { - return nil, err - } - return pag, nil + return &set, nil } func newSet(cfg *common.Config) (set, error) { @@ -156,9 +102,12 @@ func newSet(cfg *common.Config) (set, error) { }, nil } -func (set *set) runSet(ctx transformContext, transformable *transformable) error { +func (set *set) run(ctx transformContext, transformable *transformable) (*transformable, error) { value := set.value.Execute(ctx, transformable, set.defaultValue) - return set.run(ctx, transformable, set.targetInfo.Name, value) + if err := set.runFunc(ctx, transformable, set.targetInfo.Name, value); err != nil { + return nil, err + } + return transformable, nil } func setToCommonMap(m common.MapStr, key, val string) error { @@ -191,6 +140,6 @@ func setURLValue(ctx transformContext, transformable *transformable, _, value st return err } url.RawQuery = query - transformable.url = url + transformable.url = *url return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index 9e5d09d56b0..825e7f0f04e 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -6,8 +6,11 @@ package v2 import ( "bytes" + "fmt" "text/template" "time" + + "github.com/elastic/beats/v7/libbeat/common" ) type valueTpl struct { @@ -36,21 +39,28 @@ func (t *valueTpl) Unpack(in string) error { func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal string) (val string) { defer func() { if r := recover(); r != nil { + err, _ := r.(error) + fmt.Println(err) + _ = err // TODO: find alternative to this ugliness val = defaultVal } }() buf := new(bytes.Buffer) - data := map[string]interface{}{ - "header": tr.header.Clone(), - "body": tr.body.Clone(), - "url.value": tr.url.String(), - "url.params": tr.url.Query(), - "cursor": trCtx.cursor.Clone(), - "last_event": trCtx.lastEvent.Clone(), - "last_response": trCtx.lastResponse.Clone(), - } + data := common.MapStr{} + + _, _ = data.Put("header", tr.header.Clone()) + _, _ = data.Put("body", tr.body.Clone()) + _, _ = data.Put("url.value", tr.url.String()) + _, _ = data.Put("url.params", tr.url.Query()) + _, _ = data.Put("cursor", trCtx.cursor.Clone()) + _, _ = data.Put("last_event", trCtx.lastEvent.Clone()) + _, _ = data.Put("last_response.body", trCtx.lastResponse.body.Clone()) + _, _ = data.Put("last_response.header", trCtx.lastResponse.header.Clone()) + _, _ = data.Put("last_response.url.value", trCtx.lastResponse.url.String()) + _, _ = data.Put("last_response.url.params", trCtx.lastResponse.url.Query()) + if err := t.Template.Execute(buf, data); err != nil { return defaultVal } diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go new file mode 100644 index 00000000000..3d40c8b3dc8 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go @@ -0,0 +1,65 @@ +package v2 + +import ( + "net/http" + "testing" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func TestValueTpl(t *testing.T) { + cases := []struct { + name string + value string + paramCtx transformContext + paramTr *transformable + paramDefVal string + expected string + }{ + { + name: "canRenderValuesFromCtx", + value: "{{.last_response.body.param}}", + paramCtx: transformContext{ + lastResponse: newTransformable(common.MapStr{"param": 25}, nil, ""), + }, + paramTr: emptyTransformable(), + paramDefVal: "", + expected: "25", + }, + { + name: "canRenderDefaultValue", + value: "{{.last_response.body.does_not_exist}}", + paramCtx: transformContext{ + lastResponse: emptyTransformable(), + }, + paramTr: emptyTransformable(), + paramDefVal: "25", + expected: "25", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + tpl := &valueTpl{} + assert.NoError(t, tpl.Unpack(tc.value)) + got := tpl.Execute(tc.paramCtx, tc.paramTr, tc.paramDefVal) + assert.Equal(t, tc.expected, got) + }) + } +} + +func newTransformable(body common.MapStr, header http.Header, url string) *transformable { + tr := emptyTransformable() + if len(body) > 0 { + tr.body = body + } + if len(header) > 0 { + tr.header = header + } + if url != "" { + tr.url = newURL(url) + } + return tr +} From bce801681e760126883e6efd88e37bf0b1b42cc2 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 30 Oct 2020 15:31:24 +0100 Subject: [PATCH 10/35] Publish the events --- x-pack/filebeat/input/httpjson/v2/input.go | 11 ++++++++--- x-pack/filebeat/input/httpjson/v2/response.go | 10 ++++++++-- x-pack/filebeat/input/httpjson/v2/transform.go | 5 ----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go index 9923540795e..9c0ba373d97 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -6,6 +6,7 @@ package v2 import ( "context" + "encoding/json" "fmt" "net" "net/http" @@ -189,17 +190,21 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC return client.StandardClient(), nil } -func makeEvent(body string) beat.Event { +func makeEvent(body common.MapStr) (beat.Event, error) { + bodyBytes, err := json.Marshal(body) + if err != nil { + return beat.Event{}, err + } now := timeNow() fields := common.MapStr{ "event": common.MapStr{ "created": now, }, - "message": body, + "message": string(bodyBytes), } return beat.Event{ Timestamp: now, Fields: fields, - } + }, nil } diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index b8c6c4c1de3..25b7a023aa1 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -61,14 +61,20 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans } } - if rp.splitTransform == nil { + if rp.splitTransform != nil { + if err := rp.splitTransform.run(trCtx, page, ch); err != nil { + ch <- maybeEvent{err: err} + return + } continue } - if err := rp.splitTransform.run(trCtx, page, ch); err != nil { + event, err := makeEvent(page) + if err != nil { ch <- maybeEvent{err: err} return } + ch <- maybeEvent{event: event} } }() diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go index 0ba0d5263b4..55fccf3a699 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -72,11 +72,6 @@ type splitTransform interface { run(transformContext, *transformable, <-chan maybeEvent) error } -type cursorTransform interface { - transform - run(transformContext, common.MapStr) (common.MapStr, error) -} - // newTransformsFromConfig creates a list of transforms from a list of free user configurations. func newTransformsFromConfig(config transformsConfig, namespace string) (transforms, error) { var trans transforms From 52d320e8a2eb0f66769332605b5abf7d5388ed91 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 30 Oct 2020 16:28:51 +0100 Subject: [PATCH 11/35] Add check redirect and oauth tests --- x-pack/filebeat/input/httpjson/v2/config.go | 11 ++- .../filebeat/input/httpjson/v2/config_auth.go | 26 +++--- .../input/httpjson/v2/config_oauth_test.go | 81 +++++++++++++++++++ .../input/httpjson/v2/config_request.go | 20 +++-- x-pack/filebeat/input/httpjson/v2/input.go | 27 ++++++- x-pack/filebeat/input/httpjson/v2/response.go | 2 +- ...ransfrom_append.go => transform_append.go} | 0 ...ransfrom_delete.go => transform_delete.go} | 0 .../v2/{transfrom_set.go => transform_set.go} | 0 .../input/httpjson/v2/transfrom_split.go | 5 -- 10 files changed, 142 insertions(+), 30 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/v2/config_oauth_test.go rename x-pack/filebeat/input/httpjson/v2/{transfrom_append.go => transform_append.go} (100%) rename x-pack/filebeat/input/httpjson/v2/{transfrom_delete.go => transform_delete.go} (100%) rename x-pack/filebeat/input/httpjson/v2/{transfrom_set.go => transform_set.go} (100%) delete mode 100644 x-pack/filebeat/input/httpjson/v2/transfrom_split.go diff --git a/x-pack/filebeat/input/httpjson/v2/config.go b/x-pack/filebeat/input/httpjson/v2/config.go index da777d8c3bb..35056e44eca 100644 --- a/x-pack/filebeat/input/httpjson/v2/config.go +++ b/x-pack/filebeat/input/httpjson/v2/config.go @@ -29,8 +29,15 @@ func defaultConfig() config { Interval: time.Minute, Auth: &authConfig{}, Request: &requestConfig{ - Timeout: &timeout, - Method: "GET", + Timeout: &timeout, + Method: "GET", + RedirectHeadersForward: true, + RedirectLocationTrusted: false, + RedirectHeadersBanList: []string{ + "WWW-Authenticate", + "Authorization", + }, + RedirectMaxRedirects: 10, }, Response: &responseConfig{}, } diff --git a/x-pack/filebeat/input/httpjson/v2/config_auth.go b/x-pack/filebeat/input/httpjson/v2/config_auth.go index ed7a23e9caf..b9e9d3ad0da 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_auth.go +++ b/x-pack/filebeat/input/httpjson/v2/config_auth.go @@ -105,14 +105,14 @@ func (o *oAuth2Config) isEnabled() bool { func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { ctx = context.WithValue(ctx, oauth2.HTTPClient, client) - switch o.GetProvider() { + switch o.getProvider() { case oAuth2ProviderAzure, oAuth2ProviderDefault: creds := clientcredentials.Config{ ClientID: o.ClientID, ClientSecret: o.ClientSecret, - TokenURL: o.GetTokenURL(), + TokenURL: o.getTokenURL(), Scopes: o.Scopes, - EndpointParams: o.GetEndpointParams(), + EndpointParams: o.getEndpointParams(), } return creds.Client(ctx), nil case oAuth2ProviderGoogle: @@ -135,9 +135,9 @@ func (o *oAuth2Config) client(ctx context.Context, client *http.Client) (*http.C } } -// GetTokenURL returns the TokenURL. -func (o *oAuth2Config) GetTokenURL() string { - switch o.GetProvider() { +// getTokenURL returns the TokenURL. +func (o *oAuth2Config) getTokenURL() string { + switch o.getProvider() { case oAuth2ProviderAzure: if o.TokenURL == "" { return endpoints.AzureAD(o.AzureTenantID).TokenURL @@ -147,14 +147,14 @@ func (o *oAuth2Config) GetTokenURL() string { return o.TokenURL } -// GetProvider returns provider in its canonical form. -func (o oAuth2Config) GetProvider() oAuth2Provider { +// getProvider returns provider in its canonical form. +func (o oAuth2Config) getProvider() oAuth2Provider { return o.Provider.canonical() } -// GetEndpointParams returns endpoint params with any provider ones combined. -func (o oAuth2Config) GetEndpointParams() map[string][]string { - switch o.GetProvider() { +// getEndpointParams returns endpoint params with any provider ones combined. +func (o oAuth2Config) getEndpointParams() map[string][]string { + switch o.getProvider() { case oAuth2ProviderAzure: if o.AzureResource != "" { if o.EndpointParams == nil { @@ -173,7 +173,7 @@ func (o *oAuth2Config) Validate() error { return nil } - switch o.GetProvider() { + switch o.getProvider() { case oAuth2ProviderAzure: return o.validateAzureProvider() case oAuth2ProviderGoogle: @@ -183,7 +183,7 @@ func (o *oAuth2Config) Validate() error { return errors.New("both token_url and client credentials must be provided") } default: - return fmt.Errorf("unknown provider %q", o.GetProvider()) + return fmt.Errorf("unknown provider %q", o.getProvider()) } return nil diff --git a/x-pack/filebeat/input/httpjson/v2/config_oauth_test.go b/x-pack/filebeat/input/httpjson/v2/config_oauth_test.go new file mode 100644 index 00000000000..d495bec7283 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/config_oauth_test.go @@ -0,0 +1,81 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProviderCanonical(t *testing.T) { + const ( + a oAuth2Provider = "gOoGle" + b oAuth2Provider = "google" + ) + + assert.Equal(t, a.canonical(), b.canonical()) +} + +func TestGetProviderIsCanonical(t *testing.T) { + const expected oAuth2Provider = "google" + + oauth2 := oAuth2Config{Provider: "GOogle"} + assert.Equal(t, expected, oauth2.getProvider()) +} + +func TestIsEnabled(t *testing.T) { + oauth2 := oAuth2Config{} + if !oauth2.isEnabled() { + t.Fatal("OAuth2 should be enabled by default") + } + + var enabled = false + oauth2.Enabled = &enabled + + assert.False(t, oauth2.isEnabled()) + + enabled = true + + assert.True(t, oauth2.isEnabled()) +} + +func TestGetTokenURL(t *testing.T) { + const expected = "http://localhost" + oauth2 := oAuth2Config{TokenURL: "http://localhost"} + assert.Equal(t, expected, oauth2.getTokenURL()) +} + +func TestGetTokenURLWithAzure(t *testing.T) { + const expectedWithoutTenantID = "http://localhost" + oauth2 := oAuth2Config{TokenURL: "http://localhost", Provider: "azure"} + + assert.Equal(t, expectedWithoutTenantID, oauth2.getTokenURL()) + + oauth2.TokenURL = "" + oauth2.AzureTenantID = "a_tenant_id" + const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" + + assert.Equal(t, expectedWithTenantID, oauth2.getTokenURL()) + +} + +func TestGetEndpointParams(t *testing.T) { + var expected = map[string][]string{"foo": {"bar"}} + oauth2 := oAuth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} + assert.Equal(t, expected, oauth2.getEndpointParams()) +} + +func TestGetEndpointParamsWithAzure(t *testing.T) { + var expectedWithoutResource = map[string][]string{"foo": {"bar"}} + oauth2 := oAuth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} + + assert.Equal(t, expectedWithoutResource, oauth2.getEndpointParams()) + + oauth2.AzureResource = "baz" + var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} + + assert.Equal(t, expectedWithResource, oauth2.getEndpointParams()) +} diff --git a/x-pack/filebeat/input/httpjson/v2/config_request.go b/x-pack/filebeat/input/httpjson/v2/config_request.go index 95d2a7bc27e..855edc26977 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/v2/config_request.go @@ -84,14 +84,18 @@ func (u *urlConfig) Unpack(in string) error { } type requestConfig struct { - URL *urlConfig `config:"url" validate:"required"` - Method string `config:"method" validate:"required"` - Body *common.MapStr `config:"body"` - Timeout *time.Duration `config:"timeout"` - SSL *tlscommon.Config `config:"ssl"` - Retry retryConfig `config:"retry"` - RateLimit *rateLimitConfig `config:"rate_limit"` - Transforms transformsConfig `config:"transforms"` + URL *urlConfig `config:"url" validate:"required"` + Method string `config:"method" validate:"required"` + Body *common.MapStr `config:"body"` + Timeout *time.Duration `config:"timeout"` + SSL *tlscommon.Config `config:"ssl"` + Retry retryConfig `config:"retry"` + RedirectHeadersForward bool `config:"redirect.headers.forward"` + RedirectHeadersBanList []string `config:"redirect.headers.ban_list"` + RedirectLocationTrusted bool `config:"redirect.location_trusted"` + RedirectMaxRedirects int `config:"redirect.max_redirects"` + RateLimit *rateLimitConfig `config:"rate_limit"` + Transforms transformsConfig `config:"transforms"` } func (c requestConfig) getTimeout() time.Duration { diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go index 9c0ba373d97..e00fbda0ef4 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -173,7 +173,8 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC TLSClientConfig: tlsConfig.ToConfig(), DisableKeepAlives: true, }, - Timeout: timeout, + Timeout: timeout, + CheckRedirect: checkRedirect(config.Request), }, Logger: newRetryLogger(), RetryWaitMin: config.Request.Retry.getWaitMin(), @@ -190,6 +191,30 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC return client.StandardClient(), nil } +func checkRedirect(config *requestConfig) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= config.RedirectMaxRedirects { + return fmt.Errorf("stopped after %d redirects", config.RedirectMaxRedirects) + } + + if !config.RedirectHeadersForward || len(via) == 0 { + return nil + } + + prev := via[len(via)-1] // previous request to get headers from + + req.Header = prev.Header.Clone() + + if !config.RedirectLocationTrusted { + for _, k := range config.RedirectHeadersBanList { + req.Header.Del(k) + } + } + + return nil + } +} + func makeEvent(body common.MapStr) (beat.Event, error) { bodyBytes, err := json.Marshal(body) if err != nil { diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index 25b7a023aa1..8c9402d52c9 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -69,7 +69,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans continue } - event, err := makeEvent(page) + event, err := makeEvent(page.body) if err != nil { ch <- maybeEvent{err: err} return diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_append.go b/x-pack/filebeat/input/httpjson/v2/transform_append.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transfrom_append.go rename to x-pack/filebeat/input/httpjson/v2/transform_append.go diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_delete.go b/x-pack/filebeat/input/httpjson/v2/transform_delete.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transfrom_delete.go rename to x-pack/filebeat/input/httpjson/v2/transform_delete.go diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_set.go b/x-pack/filebeat/input/httpjson/v2/transform_set.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transfrom_set.go rename to x-pack/filebeat/input/httpjson/v2/transform_set.go diff --git a/x-pack/filebeat/input/httpjson/v2/transfrom_split.go b/x-pack/filebeat/input/httpjson/v2/transfrom_split.go deleted file mode 100644 index d4a74a89e77..00000000000 --- a/x-pack/filebeat/input/httpjson/v2/transfrom_split.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 From d92f88dbc8a7dd283bf32c8facb43680ab35f122 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 30 Oct 2020 16:42:13 +0100 Subject: [PATCH 12/35] Deprecate old httpjson and include both --- x-pack/filebeat/input/default-inputs/inputs.go | 2 ++ x-pack/filebeat/input/httpjson/input.go | 2 +- x-pack/filebeat/input/httpjson/v2/input.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 01e9b8ba137..56417f1d800 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" httpjsonv2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" @@ -28,6 +29,7 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 return []v2.Plugin{ cloudfoundry.Plugin(), http_endpoint.Plugin(), + httpjson.Plugin(log, store), httpjsonv2.Plugin(log, store), o365audit.Plugin(log, store), s3.Plugin(), diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 5445197f563..4df5bc80c7d 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -70,7 +70,7 @@ func Plugin(log *logp.Logger, store cursor.StateStore) v2.Plugin { return v2.Plugin{ Name: inputName, Stability: feature.Beta, - Deprecated: false, + Deprecated: true, Manager: inputManager{ stateless: &sim, cursor: &cursor.InputManager{ diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/v2/input.go index e00fbda0ef4..cd93ee3cec8 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/v2/input.go @@ -30,7 +30,7 @@ import ( ) const ( - inputName = "httpjson" + inputName = "httpjsonv2" ) var ( From 0a28dc9d7e453ebd1aafb5f5bda55500bce2d4b5 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 30 Oct 2020 17:02:53 +0100 Subject: [PATCH 13/35] Return default value if execution is empty --- x-pack/filebeat/input/httpjson/v2/value_tpl.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index 825e7f0f04e..b9ce143f1ca 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -65,7 +65,11 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal return defaultVal } - return buf.String() + val = buf.String() + if val == "" { + val = defaultVal + } + return val } var ( From 38741641cd04f374a4bce4f9e60f28484081aa8c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 2 Nov 2020 12:48:29 +0100 Subject: [PATCH 14/35] wip --- x-pack/filebeat/filebeat.yml | 299 +++--------------- .../input/httpjson/v2/config_response.go | 43 +++ x-pack/filebeat/input/httpjson/v2/response.go | 15 +- .../filebeat/input/httpjson/v2/transform.go | 13 +- .../input/httpjson/v2/transform_split.go | 153 +++++++++ 5 files changed, 263 insertions(+), 260 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/v2/transform_split.go diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 390305dd34b..5ee3119091e 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -18,253 +18,52 @@ filebeat.inputs: # you can use different inputs for various configurations. # Below are the input specific configurations. -- type: log - - # Change to true to enable this input configuration. - enabled: false - - # Paths that should be crawled and fetched. Glob based paths. - paths: - - /var/log/*.log - #- c:\programdata\elasticsearch\logs\* - - # Exclude lines. A list of regular expressions to match. It drops the lines that are - # matching any regular expression from the list. - #exclude_lines: ['^DBG'] - - # Include lines. A list of regular expressions to match. It exports the lines that are - # matching any regular expression from the list. - #include_lines: ['^ERR', '^WARN'] - - # Exclude files. A list of regular expressions to match. Filebeat drops the files that - # are matching any regular expression from the list. By default, no files are dropped. - #exclude_files: ['.gz$'] - - # Optional additional fields. These fields can be freely picked - # to add additional information to the crawled log files for filtering - #fields: - # level: debug - # review: 1 - - ### Multiline options - - # Multiline can be used for log messages spanning multiple lines. This is common - # for Java Stack Traces or C-Line Continuation - - # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [ - #multiline.pattern: ^\[ - - # Defines if the pattern set under pattern should be negated or not. Default is false. - #multiline.negate: false - - # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern - # that was (not) matched before or after or as long as a pattern is not matched based on negate. - # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash - #multiline.match: after - -# filestream is an experimental input. It is going to replace log input in the future. -- type: filestream - - # Change to true to enable this input configuration. - enabled: false - - # Paths that should be crawled and fetched. Glob based paths. - paths: - - /var/log/*.log - #- c:\programdata\elasticsearch\logs\* - - # Exclude lines. A list of regular expressions to match. It drops the lines that are - # matching any regular expression from the list. - #exclude_lines: ['^DBG'] - - # Include lines. A list of regular expressions to match. It exports the lines that are - # matching any regular expression from the list. - #include_lines: ['^ERR', '^WARN'] - - # Exclude files. A list of regular expressions to match. Filebeat drops the files that - # are matching any regular expression from the list. By default, no files are dropped. - #prospector.scanner.exclude_files: ['.gz$'] - - # Optional additional fields. These fields can be freely picked - # to add additional information to the crawled log files for filtering - #fields: - # level: debug - # review: 1 - -# ============================== Filebeat modules ============================== - -filebeat.config.modules: - # Glob pattern for configuration loading - path: ${path.config}/modules.d/*.yml - - # Set to true to enable config reloading - reload.enabled: false - - # Period on which files under path should be checked for changes - #reload.period: 10s - -# ======================= Elasticsearch template setting ======================= - -setup.template.settings: - index.number_of_shards: 1 - #index.codec: best_compression - #_source.enabled: false - - -# ================================== General =================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. -#tags: ["service-X", "web-tier"] - -# Optional fields that you can specify to add additional information to the -# output. -#fields: -# env: staging - -# ================================= Dashboards ================================= -# These settings control loading the sample dashboards to the Kibana index. Loading -# the dashboards is disabled by default and can be enabled either by setting the -# options here or by using the `setup` command. -#setup.dashboards.enabled: false - -# The URL from where to download the dashboards archive. By default this URL -# has a value which is computed based on the Beat name and version. For released -# versions, this URL points to the dashboard archive on the artifacts.elastic.co -# website. -#setup.dashboards.url: - -# =================================== Kibana =================================== - -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. -# This requires a Kibana endpoint configuration. -setup.kibana: - - # Kibana Host - # Scheme and port can be left out and will be set to the default (http and 5601) - # In case you specify and additional path, the scheme is required: http://localhost:5601/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 - #host: "localhost:5601" - - # Kibana Space ID - # ID of the Kibana Space into which the dashboards should be loaded. By default, - # the Default Space will be used. - #space.id: - -# =============================== Elastic Cloud ================================ - -# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). - -# The cloud.id setting overwrites the `output.elasticsearch.hosts` and -# `setup.kibana.host` options. -# You can find the `cloud.id` in the Elastic Cloud web UI. -#cloud.id: - -# The cloud.auth setting overwrites the `output.elasticsearch.username` and -# `output.elasticsearch.password` settings. The format is `:`. -#cloud.auth: - -# ================================== Outputs =================================== - -# Configure what output to use when sending the data collected by the beat. - -# ---------------------------- Elasticsearch Output ---------------------------- -output.elasticsearch: - # Array of hosts to connect to. - hosts: ["localhost:9200"] - - # Protocol - either `http` (default) or `https`. - #protocol: "https" - - # Authentication credentials - either API key or username/password. - #api_key: "id:api_key" - #username: "elastic" - #password: "changeme" - -# ------------------------------ Logstash Output ------------------------------- -#output.logstash: - # The Logstash hosts - #hosts: ["localhost:5044"] - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - -# ================================= Processors ================================= -processors: - - add_host_metadata: - when.not.contains.tags: forwarded - - add_cloud_metadata: ~ - - add_docker_metadata: ~ - - add_kubernetes_metadata: ~ - -# ================================== Logging =================================== - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: debug - -# At debug level, you can selectively enable logging only for some components. -# To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publisher", "service". -#logging.selectors: ["*"] - -# ============================= X-Pack Monitoring ============================== -# Filebeat can export internal metrics to a central Elasticsearch monitoring -# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The -# reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#monitoring.enabled: false - -# Sets the UUID of the Elasticsearch cluster under which monitoring data for this -# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch -# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. -#monitoring.cluster_uuid: - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. -# Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the Elasticsearch -# output configuration, so if you have the Elasticsearch output configured such -# that it is pointing to your Elasticsearch monitoring cluster, you can simply -# uncomment the following line. -#monitoring.elasticsearch: - -# ============================== Instrumentation =============================== - -# Instrumentation support for the filebeat. -#instrumentation: - # Set to true to enable instrumentation of filebeat. - #enabled: false - - # Environment in which filebeat is running on (eg: staging, production, etc.) - #environment: "" - - # APM Server hosts to report instrumentation results to. - #hosts: - # - http://localhost:8200 - - # API Key for the APM Server(s). - # If api_key is set then secret_token will be ignored. - #api_key: - - # Secret token for the APM Server(s). - #secret_token: - - -# ================================= Migration ================================== - -# This allows to enable 6.7 migration aliases -#migration.6_to_7.enabled: true - +- type: httpjsonv2 + interval: 10s + request.method: post + request.url: https://patata.free.beeceptor.com + request.transforms: + - set: + target: body.foo + value: bazz + response.transforms: + - set: + target: body.foo + value: patata + response.pagination: + - set: + target: url.value + value: "https://patata.free.beeceptor.com/page" + - set: + target: url.params.p + value: "{{.last_response.body.page}}" + response.split: + target: body.foo + type: array|map + transforms: + - ... + split: + + +# limit split to be last (and to be just one) +# split can't have split transform itself +# iterate over maps or arrays dynamically +# keys_field to keep value of the key if a map is found +# add option to keep headers between redirects + +# { +# "bar": [ +# { +# "bazz": { +# "somekey": [ +# "bazinga": { +# "key": "value" +# } +# ] +# } +# } +# ] +# } + +output.console: + pretty: true diff --git a/x-pack/filebeat/input/httpjson/v2/config_response.go b/x-pack/filebeat/input/httpjson/v2/config_response.go index e42f8c8b878..b1b81081200 100644 --- a/x-pack/filebeat/input/httpjson/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/v2/config_response.go @@ -4,9 +4,29 @@ package v2 +import ( + "fmt" + "strings" +) + +const ( + splitTypeArr = "array" + splitTypeMap = "map" +) + type responseConfig struct { Transforms transformsConfig `config:"transforms"` Pagination transformsConfig `config:"pagination"` + Split *splitConfig `config:"split"` +} + +type splitConfig struct { + Target string `config:"target" validation:"required"` + Type string `config:"type"` + Transforms transformsConfig `config:"transforms"` + Split *splitConfig `config:"split"` + KeepParent bool `config:"keep_parent"` + KeyField string `config:"key_field"` } func (c *responseConfig) Validate() error { @@ -18,3 +38,26 @@ func (c *responseConfig) Validate() error { } return nil } + +func (c *splitConfig) Validate() error { + if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace); err != nil { + return err + } + + c.Type = strings.ToLower(c.Type) + switch c.Type { + case "", splitTypeArr: + if c.KeyField != "" { + return fmt.Errorf("key_field can only be used with a %s split type", splitTypeMap) + } + case splitTypeMap: + default: + return fmt.Errorf("invalid split type: %s", c.Type) + } + + if _, err := newSplitResponse(c); err != nil { + return err + } + + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index 8c9402d52c9..b97a99ea257 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -18,9 +18,9 @@ func registerResponseTransforms() { } type responseProcessor struct { - splitTransform splitTransform - transforms []basicTransform - pagination *pagination + transforms []basicTransform + split *split + pagination *pagination } func newResponseProcessor(config *responseConfig, pagination *pagination) *responseProcessor { @@ -32,6 +32,11 @@ func newResponseProcessor(config *responseConfig, pagination *pagination) *respo } ts, _ := newBasicTransformsFromConfig(config.Transforms, responseNamespace) rp.transforms = ts + + split, _ := newSplitResponse(config.Split) + + rp.split = split + return rp } @@ -61,8 +66,8 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans } } - if rp.splitTransform != nil { - if err := rp.splitTransform.run(trCtx, page, ch); err != nil { + if rp.split != nil { + if err := rp.split.run(trCtx, page, ch); err != nil { ch <- maybeEvent{err: err} return } diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go index 55fccf3a699..37db615ca0e 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -42,6 +42,14 @@ type transformable struct { url url.URL } +func (t *transformable) clone() *transformable { + return &transformable{ + body: t.body.Clone(), + header: t.header.Clone(), + url: t.url, + } +} + func emptyTransformable() *transformable { return &transformable{ body: make(common.MapStr), @@ -67,11 +75,6 @@ func (e maybeEvent) failed() bool { return e.err != nil } func (e maybeEvent) Error() string { return e.err.Error() } -type splitTransform interface { - transform - run(transformContext, *transformable, <-chan maybeEvent) error -} - // newTransformsFromConfig creates a list of transforms from a list of free user configurations. func newTransformsFromConfig(config transformsConfig, namespace string) (transforms, error) { var trans transforms diff --git a/x-pack/filebeat/input/httpjson/v2/transform_split.go b/x-pack/filebeat/input/httpjson/v2/transform_split.go new file mode 100644 index 00000000000..724af3d82ae --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/transform_split.go @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "errors" + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" +) + +type split struct { + targetInfo targetInfo + kind string + transforms []basicTransform + split *split + keepParent bool + keyField string +} + +func newSplitResponse(cfg *splitConfig) (*split, error) { + if cfg == nil { + return nil, nil + } + + split, err := newSplit(cfg) + if err != nil { + return nil, err + } + + if split.targetInfo.Type != targetBody { + return nil, fmt.Errorf("invalid target type: %s", split.targetInfo.Type) + } + + return split, nil +} + +func newSplit(c *splitConfig) (*split, error) { + ti, err := getTargetInfo(c.Target) + if err != nil { + return nil, err + } + + ts, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace) + if err != nil { + return nil, err + } + + var s *split + if c.Split != nil { + s, err = newSplitResponse(c.Split) + if err != nil { + return nil, err + } + } + + return &split{ + targetInfo: ti, + kind: c.Type, + keepParent: c.KeepParent, + keyField: c.KeyField, + transforms: ts, + split: s, + }, nil +} + +func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEvent) error { + respCpy := resp.clone() + var err error + for _, t := range s.transforms { + respCpy, err = t.run(ctx, respCpy) + if err != nil { + return err + } + } + + v, err := respCpy.body.GetValue(s.targetInfo.Name) + if err != nil && err != common.ErrKeyNotFound { + return err + } + + switch s.kind { + case splitTypeArr: + arr, ok := v.([]interface{}) + if !ok { + return fmt.Errorf("field %s needs to be an array to be able to split on it", s.targetInfo.Name) + } + for _, a := range arr { + m, ok := a.(map[string]interface{}) + if !ok { + // TODO + } + + if s.keepParent { + _, _ = respCpy.body.Put(s.targetInfo.Name, m) + } else { + respCpy.body = common.MapStr(m) + } + + if s.split != nil { + return s.split.run(ctx, respCpy, ch) + } + + event, err := makeEvent(respCpy.body) + if err != nil { + return err + } + + ch <- maybeEvent{event: event} + } + + return nil + case splitTypeMap: + m, ok := v.(map[string]interface{}) + if !ok { + return fmt.Errorf("field %s needs to be a map to be able to split on it", s.targetInfo.Name) + } + for k, mm := range m { + v, ok := mm.(map[string]interface{}) + if !ok { + // TODO + } + + vv := common.MapStr(v) + if s.keyField != "" { + _, _ = vv.Put(s.keyField, k) + } + + if s.keepParent { + _, _ = respCpy.body.Put(s.targetInfo.Name, vv) + } else { + respCpy.body = vv + } + + if s.split != nil { + return s.split.run(ctx, respCpy, ch) + } + + event, err := makeEvent(respCpy.body) + if err != nil { + return err + } + + ch <- maybeEvent{event: event} + } + + return nil + } + + return errors.New("invalid split type") +} From cb2b24cc6206bac6e4e149c70131feb94e42fbc8 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 2 Nov 2020 17:20:38 +0100 Subject: [PATCH 15/35] Add split --- .../v2/{transform_split.go => split.go} | 83 ++++---- .../filebeat/input/httpjson/v2/split_test.go | 180 ++++++++++++++++++ 2 files changed, 225 insertions(+), 38 deletions(-) rename x-pack/filebeat/input/httpjson/v2/{transform_split.go => split.go} (68%) create mode 100644 x-pack/filebeat/input/httpjson/v2/split_test.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_split.go b/x-pack/filebeat/input/httpjson/v2/split.go similarity index 68% rename from x-pack/filebeat/input/httpjson/v2/transform_split.go rename to x-pack/filebeat/input/httpjson/v2/split.go index 724af3d82ae..ee0f29c7e9a 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform_split.go +++ b/x-pack/filebeat/input/httpjson/v2/split.go @@ -87,63 +87,36 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEv if !ok { return fmt.Errorf("field %s needs to be an array to be able to split on it", s.targetInfo.Name) } + for _, a := range arr { - m, ok := a.(map[string]interface{}) + m, ok := toMapStr(a) if !ok { - // TODO - } - - if s.keepParent { - _, _ = respCpy.body.Put(s.targetInfo.Name, m) - } else { - respCpy.body = common.MapStr(m) + return errors.New("split can only be applied on object lists") } - if s.split != nil { - return s.split.run(ctx, respCpy, ch) - } - - event, err := makeEvent(respCpy.body) - if err != nil { + if err := s.sendEvent(ctx, respCpy, m, ch); err != nil { return err } - - ch <- maybeEvent{event: event} } return nil case splitTypeMap: - m, ok := v.(map[string]interface{}) + ms, ok := toMapStr(v) if !ok { return fmt.Errorf("field %s needs to be a map to be able to split on it", s.targetInfo.Name) } - for k, mm := range m { - v, ok := mm.(map[string]interface{}) + + for k, v := range ms { + m, ok := toMapStr(v) if !ok { - // TODO + return errors.New("split can only be applied on object lists") } - - vv := common.MapStr(v) if s.keyField != "" { - _, _ = vv.Put(s.keyField, k) - } - - if s.keepParent { - _, _ = respCpy.body.Put(s.targetInfo.Name, vv) - } else { - respCpy.body = vv + _, _ = m.Put(s.keyField, k) } - - if s.split != nil { - return s.split.run(ctx, respCpy, ch) - } - - event, err := makeEvent(respCpy.body) - if err != nil { + if err := s.sendEvent(ctx, respCpy, m, ch); err != nil { return err } - - ch <- maybeEvent{event: event} } return nil @@ -151,3 +124,37 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEv return errors.New("invalid split type") } + +func toMapStr(v interface{}) (common.MapStr, bool) { + var m common.MapStr + switch ts := v.(type) { + case common.MapStr: + m = ts + case map[string]interface{}: + m = common.MapStr(ts) + default: + return nil, false + } + return m, true +} + +func (s *split) sendEvent(ctx transformContext, resp *transformable, m common.MapStr, ch chan<- maybeEvent) error { + if s.keepParent { + _, _ = resp.body.Put(s.targetInfo.Name, m) + } else { + resp.body = m + } + + if s.split != nil { + return s.split.run(ctx, resp, ch) + } + + event, err := makeEvent(resp.body) + if err != nil { + return err + } + + ch <- maybeEvent{event: event} + + return nil +} diff --git a/x-pack/filebeat/input/httpjson/v2/split_test.go b/x-pack/filebeat/input/httpjson/v2/split_test.go new file mode 100644 index 00000000000..74d262cb1eb --- /dev/null +++ b/x-pack/filebeat/input/httpjson/v2/split_test.go @@ -0,0 +1,180 @@ +package v2 + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func TestSplit(t *testing.T) { + cases := []struct { + name string + config *splitConfig + ctx transformContext + resp *transformable + expectedMessages []string + expectedErr error + }{ + { + name: "Two nested Split Arrays with keep_parent", + config: &splitConfig{ + Target: "body.alerts", + Type: "array", + KeepParent: true, + Split: &splitConfig{ + Target: "body.alerts.entities", + Type: "array", + KeepParent: true, + }, + }, + ctx: emptyTransformContext(), + resp: &transformable{ + body: common.MapStr{ + "this": "is kept", + "alerts": []interface{}{ + map[string]interface{}{ + "this_is": "also kept", + "entities": []interface{}{ + map[string]interface{}{ + "something": "something", + }, + map[string]interface{}{ + "else": "else", + }, + }, + }, + map[string]interface{}{ + "this_is": "also kept 2", + "entities": []interface{}{ + map[string]interface{}{ + "something": "something 2", + }, + map[string]interface{}{ + "else": "else 2", + }, + }, + }, + }, + }, + }, + expectedMessages: []string{ + `{ + "this": "is kept", + "alerts": { + "this_is": "also kept", + "entities": { + "something": "something" + } + } + }`, + `{ + "this": "is kept", + "alerts": { + "this_is": "also kept", + "entities": { + "else": "else" + } + } + }`, + `{ + "this": "is kept", + "alerts": { + "this_is": "also kept 2", + "entities": { + "something": "something 2" + } + } + }`, + `{ + "this": "is kept", + "alerts": { + "this_is": "also kept 2", + "entities": { + "else": "else 2" + } + } + }`, + }, + expectedErr: nil, + }, + { + name: "A nested array with a nested map", + config: &splitConfig{ + Target: "body.alerts", + Type: "array", + KeepParent: false, + Split: &splitConfig{ + Target: "body.entities", + Type: "map", + KeepParent: true, + KeyField: "id", + }, + }, + ctx: emptyTransformContext(), + resp: &transformable{ + body: common.MapStr{ + "this": "is not kept", + "alerts": []interface{}{ + map[string]interface{}{ + "this_is": "kept", + "entities": map[string]interface{}{ + "id1": map[string]interface{}{ + "something": "else", + }, + }, + }, + map[string]interface{}{ + "this_is": "also kept", + "entities": map[string]interface{}{ + "id2": map[string]interface{}{ + "something": "else 2", + }, + }, + }, + }, + }, + }, + expectedMessages: []string{ + `{ + "this_is": "kept", + "entities": { + "id": "id1", + "something": "else" + } + }`, + `{ + "this_is": "also kept", + "entities": { + "id": "id2", + "something": "else 2" + } + }`, + }, + expectedErr: nil, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + ch := make(chan maybeEvent, len(tc.expectedMessages)) + split, err := newSplitResponse(tc.config) + assert.NoError(t, err) + err = split.run(tc.ctx, tc.resp, ch) + if tc.expectedErr == nil { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr.Error()) + } + close(ch) + assert.Equal(t, len(tc.expectedMessages), len(ch)) + for _, msg := range tc.expectedMessages { + e := <-ch + assert.NoError(t, e.err) + got := e.event.Fields["message"].(string) + assert.JSONEq(t, msg, got) + } + }) + } +} From c4d0b3300c625547a745e3139da6551ca3616f05 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 3 Nov 2020 11:51:34 +0100 Subject: [PATCH 16/35] Handle request errors and pagination end --- .../filebeat/input/httpjson/v2/pagination.go | 17 ++++++++++++-- x-pack/filebeat/input/httpjson/v2/request.go | 6 +++++ x-pack/filebeat/input/httpjson/v2/response.go | 19 +++++++++++----- x-pack/filebeat/input/httpjson/v2/split.go | 22 ++++++++++++++++--- .../filebeat/input/httpjson/v2/transform.go | 4 ++-- .../input/httpjson/v2/transform_set.go | 15 ++++++++----- .../filebeat/input/httpjson/v2/value_tpl.go | 12 +++++++++- 7 files changed, 76 insertions(+), 19 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/v2/pagination.go b/x-pack/filebeat/input/httpjson/v2/pagination.go index ea4039fb35a..28358224251 100644 --- a/x-pack/filebeat/input/httpjson/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/v2/pagination.go @@ -7,6 +7,7 @@ package v2 import ( "context" "encoding/json" + "fmt" "io/ioutil" "net/http" "net/url" @@ -33,11 +34,13 @@ func newPagination(config config, httpClient *http.Client, log *logp.Logger) *pa if config.Response == nil { return pagination } - ts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace) + + rts, _ := newBasicTransformsFromConfig(config.Request.Transforms, requestNamespace) + pts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace) requestFactory := newPaginationRequestFactory( config.Request.Method, *config.Request.URL.URL, - ts, + append(rts, pts...), config.Auth, log, ) @@ -98,6 +101,11 @@ func (iter *pageIterator) next() (*transformable, bool, error) { httpReq, err := iter.pagination.requestFactory.newHTTPRequest(iter.stdCtx, iter.trCtx) if err != nil { + if err == errNewURLValueNotSet { + // if this error happens here it means the transform used to pick the new url.value + // did not find any new value and we can stop paginating without error + return nil, false, nil + } return nil, false, err } @@ -106,6 +114,11 @@ func (iter *pageIterator) next() (*transformable, bool, error) { return nil, false, err } + if resp.StatusCode > 399 { + body, _ := ioutil.ReadAll(resp.Body) + return nil, false, fmt.Errorf("server responded with status code %d: %s", resp.StatusCode, string(body)) + } + iter.resp = resp tr, err := iter.getPage() diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/v2/request.go index b2fd3e3da22..beefbba683d 100644 --- a/x-pack/filebeat/input/httpjson/v2/request.go +++ b/x-pack/filebeat/input/httpjson/v2/request.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" @@ -143,6 +144,11 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu } defer httpResp.Body.Close() + if httpResp.StatusCode > 399 { + body, _ := ioutil.ReadAll(httpResp.Body) + return fmt.Errorf("server responded with status code %d: %s", httpResp.StatusCode, string(body)) + } + eventsCh, err := r.responseProcessor.startProcessing(stdCtx, trCtx, httpResp) if err != nil { return err diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/v2/response.go index b97a99ea257..7978263107a 100644 --- a/x-pack/filebeat/input/httpjson/v2/response.go +++ b/x-pack/filebeat/input/httpjson/v2/response.go @@ -6,6 +6,7 @@ package v2 import ( "context" + "fmt" "net/http" ) @@ -54,32 +55,38 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans return } - if !hasNext { + if !hasNext || len(page.body) == 0 { return } for _, t := range rp.transforms { page, err = t.run(trCtx, page) if err != nil { + fmt.Println("=== 2") ch <- maybeEvent{err: err} return } } - if rp.split != nil { - if err := rp.split.run(trCtx, page, ch); err != nil { + if rp.split == nil { + event, err := makeEvent(page.body) + if err != nil { ch <- maybeEvent{err: err} return } + ch <- maybeEvent{event: event} continue } - event, err := makeEvent(page.body) - if err != nil { + if err := rp.split.run(trCtx, page, ch); err != nil { + if err == errEmtpyField { + // nothing else to send + return + } + ch <- maybeEvent{err: err} return } - ch <- maybeEvent{event: event} } }() diff --git a/x-pack/filebeat/input/httpjson/v2/split.go b/x-pack/filebeat/input/httpjson/v2/split.go index ee0f29c7e9a..b0f699876a9 100644 --- a/x-pack/filebeat/input/httpjson/v2/split.go +++ b/x-pack/filebeat/input/httpjson/v2/split.go @@ -11,6 +11,8 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +var errEmtpyField = errors.New("the requested field is emtpy") + type split struct { targetInfo targetInfo kind string @@ -82,10 +84,14 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEv } switch s.kind { - case splitTypeArr: + case "", splitTypeArr: arr, ok := v.([]interface{}) if !ok { - return fmt.Errorf("field %s needs to be an array to be able to split on it", s.targetInfo.Name) + return fmt.Errorf("field %s needs to be an array to be able to split on it but it is %T", s.targetInfo.Name, v) + } + + if len(arr) == 0 { + return errEmtpyField } for _, a := range arr { @@ -101,9 +107,17 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEv return nil case splitTypeMap: + if v == nil { + return errEmtpyField + } + ms, ok := toMapStr(v) if !ok { - return fmt.Errorf("field %s needs to be a map to be able to split on it", s.targetInfo.Name) + return fmt.Errorf("field %s needs to be a map to be able to split on it but it is %T", s.targetInfo.Name, v) + } + + if len(ms) == 0 { + return errEmtpyField } for k, v := range ms { @@ -156,5 +170,7 @@ func (s *split) sendEvent(ctx transformContext, resp *transformable, m common.Ma ch <- maybeEvent{event: event} + *ctx.lastEvent = event + return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/v2/transform.go index 37db615ca0e..f1fa5397270 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/v2/transform.go @@ -24,15 +24,15 @@ type transforms []transform type transformContext struct { cursor common.MapStr - lastEvent common.MapStr + lastEvent *beat.Event lastResponse *transformable } func emptyTransformContext() transformContext { return transformContext{ cursor: make(common.MapStr), - lastEvent: make(common.MapStr), lastResponse: emptyTransformable(), + lastEvent: &beat.Event{}, } } diff --git a/x-pack/filebeat/input/httpjson/v2/transform_set.go b/x-pack/filebeat/input/httpjson/v2/transform_set.go index 79ab7e438c4..e9ce6fe1e4e 100644 --- a/x-pack/filebeat/input/httpjson/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/v2/transform_set.go @@ -2,12 +2,14 @@ package v2 import ( "fmt" - httpURL "net/url" + "net/url" "github.com/elastic/beats/v7/libbeat/common" "github.com/pkg/errors" ) +var errNewURLValueNotSet = errors.New("the new url.value was not set") + const setName = "set" type setConfig struct { @@ -134,12 +136,15 @@ func setURLParams(ctx transformContext, transformable *transformable, key, value } func setURLValue(ctx transformContext, transformable *transformable, _, value string) error { - query := transformable.url.Query().Encode() - url, err := httpURL.Parse(value) + // if the template processing did not find any value + // we fail without parsing + if value == "" || value == "" { + return errNewURLValueNotSet + } + url, err := url.Parse(value) if err != nil { - return err + return errNewURLValueNotSet } - url.RawQuery = query transformable.url = *url return nil } diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index b9ce143f1ca..cbe5857da54 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -10,6 +10,7 @@ import ( "text/template" "time" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" ) @@ -55,7 +56,7 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal _, _ = data.Put("url.value", tr.url.String()) _, _ = data.Put("url.params", tr.url.Query()) _, _ = data.Put("cursor", trCtx.cursor.Clone()) - _, _ = data.Put("last_event", trCtx.lastEvent.Clone()) + _, _ = data.Put("last_event", cloneEvent(trCtx.lastEvent)) _, _ = data.Put("last_response.body", trCtx.lastResponse.body.Clone()) _, _ = data.Put("last_response.header", trCtx.lastResponse.header.Clone()) _, _ = data.Put("last_response.url.value", trCtx.lastResponse.url.String()) @@ -72,6 +73,15 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal return val } +func cloneEvent(event *beat.Event) beat.Event { + return beat.Event{ + Timestamp: event.Timestamp, + Meta: event.Meta.Clone(), + Fields: event.Fields.Clone(), + TimeSeries: event.TimeSeries, + } +} + var ( predefinedLayouts = map[string]string{ "ANSIC": time.ANSIC, From b653aec093823ae2d5aac0c3ca15b93b02aa1010 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 5 Nov 2020 16:25:41 +0100 Subject: [PATCH 17/35] Add date functions for templates and tests --- .../filebeat/input/httpjson/v2/value_tpl.go | 96 ++++++++----- .../input/httpjson/v2/value_tpl_test.go | 134 +++++++++++++++++- 2 files changed, 191 insertions(+), 39 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index cbe5857da54..f562a6df8a5 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -6,7 +6,6 @@ package v2 import ( "bytes" - "fmt" "text/template" "time" @@ -22,10 +21,14 @@ func (t *valueTpl) Unpack(in string) error { tpl, err := template.New(""). Option("missingkey=error"). Funcs(template.FuncMap{ - "now": now, - "formatDate": formatDate, - "parseDate": parseDate, - "getRFC5988Link": getRFC5988Link, + "now": now, + "hour": hour, + "parseDate": parseDate, + "formatDate": formatDate, + "parseTimestamp": parseTimestamp, + "parseTimestampMilli": parseTimestampMilli, + "parseTimestampNano": parseTimestampNano, + "getRFC5988Link": getRFC5988Link, }). Parse(in) if err != nil { @@ -40,10 +43,6 @@ func (t *valueTpl) Unpack(in string) error { func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal string) (val string) { defer func() { if r := recover(); r != nil { - err, _ := r.(error) - fmt.Println(err) - _ = err - // TODO: find alternative to this ugliness val = defaultVal } }() @@ -74,6 +73,9 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal } func cloneEvent(event *beat.Event) beat.Event { + if event == nil { + return beat.Event{} + } return beat.Event{ Timestamp: event.Timestamp, Meta: event.Meta.Clone(), @@ -98,34 +100,30 @@ var ( } ) -func formatDate(date time.Time, layout string, tz ...string) string { - if found := predefinedLayouts[layout]; found != "" { - layout = found - } else { - layout = time.RFC3339 - } - - if len(tz) > 0 { - if loc, err := time.LoadLocation(tz[0]); err == nil { - date = date.In(loc) - } else { - date = date.UTC() - } - } else { - date = date.UTC() +func now(add ...time.Duration) time.Time { + now := timeNow() + if len(add) == 0 { + return now } + return now.Add(add[0]) +} - return date.Format(layout) +func hour(n int) time.Duration { + return time.Duration(n) * time.Hour } -func parseDate(date, layout string) time.Time { - if found := predefinedLayouts[layout]; found != "" { - layout = found +func parseDate(date string, layout ...string) time.Time { + var ly string + if len(layout) == 0 { + ly = "RFC3339" } else { - layout = time.RFC3339 + ly = layout[0] + } + if found := predefinedLayouts[ly]; found != "" { + ly = found } - t, err := time.Parse(layout, date) + t, err := time.Parse(ly, date) if err != nil { return time.Time{} } @@ -133,12 +131,40 @@ func parseDate(date, layout string) time.Time { return t } -func now(add ...time.Duration) time.Time { - now := time.Now() - if len(add) == 0 { - return now +func formatDate(date time.Time, layouttz ...string) string { + var layout, tz string + switch { + case len(layouttz) == 0: + layout = "RFC3339" + case len(layouttz) == 1: + layout = layouttz[0] + case len(layouttz) > 1: + layout, tz = layouttz[0], layouttz[1] } - return now.Add(add[0]) + + if found := predefinedLayouts[layout]; found != "" { + layout = found + } + + if loc, err := time.LoadLocation(tz); err == nil { + date = date.In(loc) + } else { + date = date.UTC() + } + + return date.Format(layout) +} + +func parseTimestamp(s int64) time.Time { + return time.Unix(s, 0) +} + +func parseTimestampMilli(ms int64) time.Time { + return time.Unix(0, ms*1e6) +} + +func parseTimestampNano(ns int64) time.Time { + return time.Unix(0, ns) } func getRFC5988Link(links, rel string) string { diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go index 3d40c8b3dc8..6026e344705 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go @@ -3,6 +3,7 @@ package v2 import ( "net/http" "testing" + "time" "github.com/elastic/beats/v7/libbeat/common" "github.com/stretchr/testify/assert" @@ -16,19 +17,21 @@ func TestValueTpl(t *testing.T) { paramTr *transformable paramDefVal string expected string + setup func() + teardown func() }{ { - name: "canRenderValuesFromCtx", + name: "can render values from ctx", value: "{{.last_response.body.param}}", paramCtx: transformContext{ - lastResponse: newTransformable(common.MapStr{"param": 25}, nil, ""), + lastResponse: newTestTransformable(common.MapStr{"param": 25}, nil, ""), }, paramTr: emptyTransformable(), paramDefVal: "", expected: "25", }, { - name: "canRenderDefaultValue", + name: "can render default value if execute fails", value: "{{.last_response.body.does_not_exist}}", paramCtx: transformContext{ lastResponse: emptyTransformable(), @@ -37,11 +40,134 @@ func TestValueTpl(t *testing.T) { paramDefVal: "25", expected: "25", }, + { + name: "can render default value if template is empty", + value: "", + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + paramDefVal: "25", + expected: "25", + }, + { + name: "can render default value if execute panics", + value: "{{.last_response.panic}}", + paramDefVal: "25", + expected: "25", + }, + { + name: "func hour", + value: `{{ hour -1 }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "-1h0m0s", + }, + { + name: "func now", + setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `{{ now }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 13:25:32 +0000 UTC", + }, + { + name: "func now with duration", + setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `{{ now (-1|hour) }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 12:25:32 +0000 UTC", + }, + { + name: "func parseDate", + value: `{{ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 12:25:32.1234567 +0000 UTC", + }, + { + name: "func parseDate defaults to RFC3339", + value: `{{ parseDate "2020-11-05T12:25:32Z" }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 12:25:32 +0000 UTC", + }, + { + name: "func parseDate with custom layout", + value: `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 12:25:32 +0000 UTC", + }, + { + name: "func formatDate", + setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `{{ formatDate (now) "UnixDate" "America/New_York" }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "Thu Nov 5 08:25:32 EST 2020", + }, + { + name: "func formatDate defaults to UTC", + setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `{{ formatDate (now) "UnixDate" }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "Thu Nov 5 13:25:32 UTC 2020", + }, + { + name: "func formatDate falls back to UTC", + setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `{{ formatDate (now) "UnixDate" "wrong/tz"}}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "Thu Nov 5 13:25:32 UTC 2020", + }, + { + name: "func parseTimestamp", + value: `{{ (parseTimestamp 1604582732).UTC }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 13:25:32 +0000 UTC", + }, + { + name: "func parseTimestampMilli", + value: `{{ (parseTimestampMilli 1604582732000).UTC }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 13:25:32 +0000 UTC", + }, + { + name: "func parseTimestampNano", + value: `{{ (parseTimestampNano 1604582732000000000).UTC }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05 13:25:32 +0000 UTC", + }, + { + name: "can execute functions pipeline", + setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `{{ -1 | hour | now | formatDate }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + expected: "2020-11-05T12:25:32Z", + }, } for _, tc := range cases { tc := tc t.Run(tc.name, func(t *testing.T) { + if tc.setup != nil { + tc.setup() + } + if tc.teardown != nil { + t.Cleanup(tc.teardown) + } tpl := &valueTpl{} assert.NoError(t, tpl.Unpack(tc.value)) got := tpl.Execute(tc.paramCtx, tc.paramTr, tc.paramDefVal) @@ -50,7 +176,7 @@ func TestValueTpl(t *testing.T) { } } -func newTransformable(body common.MapStr, header http.Header, url string) *transformable { +func newTestTransformable(body common.MapStr, header http.Header, url string) *transformable { tr := emptyTransformable() if len(body) > 0 { tr.body = body From 4aa6d9c66d2651e0cf0cb62ab2c813a45942a6b4 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 6 Nov 2020 09:57:59 +0100 Subject: [PATCH 18/35] Add getRFC5988Link functionality to templates --- .../filebeat/input/httpjson/v2/value_tpl.go | 22 +++++++++- .../input/httpjson/v2/value_tpl_test.go | 40 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/v2/value_tpl.go index f562a6df8a5..e427583c471 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl.go @@ -6,6 +6,7 @@ package v2 import ( "bytes" + "regexp" "text/template" "time" @@ -167,6 +168,25 @@ func parseTimestampNano(ns int64) time.Time { return time.Unix(0, ns) } -func getRFC5988Link(links, rel string) string { +var regexpLinkRel = regexp.MustCompile(`<(.*)>;.*\srel\="?([^;"]*)`) + +func getRFC5988Link(rel string, links []string) string { + for _, link := range links { + if !regexpLinkRel.MatchString(link) { + continue + } + + matches := regexpLinkRel.FindStringSubmatch(link) + if len(matches) != 3 { + continue + } + + if matches[2] != rel { + continue + } + + return matches[1] + } + return "" } diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go index 6026e344705..cac77ec54ae 100644 --- a/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go @@ -148,6 +148,46 @@ func TestValueTpl(t *testing.T) { paramTr: emptyTransformable(), expected: "2020-11-05 13:25:32 +0000 UTC", }, + { + name: "func getRFC5988Link", + value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, + paramCtx: transformContext{ + lastResponse: newTestTransformable( + nil, + http.Header{"Link": []string{ + `; title="Page 3"; rel="next"`, + `; title="Page 1"; rel="previous"`, + }}, + "", + ), + }, + paramTr: emptyTransformable(), + expected: "https://example.com/api/v1/users?before=00ubfjQEMYBLRUWIEDKK", + }, + { + name: "func getRFC5988Link does not match", + value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, + paramCtx: transformContext{ + lastResponse: newTestTransformable( + nil, + http.Header{"Link": []string{ + ``, + }}, + "", + ), + }, + paramTr: emptyTransformable(), + paramDefVal: "https://example.com/default", + expected: "https://example.com/default", + }, + { + name: "func getRFC5988Link empty header", + value: `{{ getRFC5988Link "previous" .last_response.header.Empty }}`, + paramCtx: emptyTransformContext(), + paramTr: emptyTransformable(), + paramDefVal: "https://example.com/default", + expected: "https://example.com/default", + }, { name: "can execute functions pipeline", setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, From 6759001c23b3e71a5a99c147363322741cf69c47 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 6 Nov 2020 10:31:02 +0100 Subject: [PATCH 19/35] Switch between v1/v2 based on config flag --- .../filebeat/input/default-inputs/inputs.go | 2 -- x-pack/filebeat/input/httpjson/input.go | 14 +++++++------ .../filebeat/input/httpjson/input_manager.go | 21 +++++++++++++------ .../httpjson/{ => internal}/v2/config.go | 0 .../httpjson/{ => internal}/v2/config_auth.go | 0 .../{ => internal}/v2/config_oauth_test.go | 0 .../{ => internal}/v2/config_request.go | 0 .../{ => internal}/v2/config_response.go | 0 .../input/httpjson/{ => internal}/v2/input.go | 19 ----------------- .../{ => internal}/v2/input_manager.go | 18 ++++++++++++---- .../{ => internal}/v2/input_stateless.go | 0 .../httpjson/{ => internal}/v2/pagination.go | 0 .../httpjson/{ => internal}/v2/request.go | 0 .../httpjson/{ => internal}/v2/response.go | 0 .../input/httpjson/{ => internal}/v2/split.go | 0 .../httpjson/{ => internal}/v2/split_test.go | 0 .../httpjson/{ => internal}/v2/transform.go | 0 .../{ => internal}/v2/transform_append.go | 0 .../{ => internal}/v2/transform_delete.go | 0 .../{ => internal}/v2/transform_registry.go | 0 .../{ => internal}/v2/transform_set.go | 0 .../{ => internal}/v2/transform_set_test.go | 0 .../{ => internal}/v2/transform_target.go | 0 .../httpjson/{ => internal}/v2/value_tpl.go | 0 .../{ => internal}/v2/value_tpl_test.go | 0 25 files changed, 37 insertions(+), 37 deletions(-) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/config.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/config_auth.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/config_oauth_test.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/config_request.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/config_response.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/input.go (91%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/input_manager.go (71%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/input_stateless.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/pagination.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/request.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/response.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/split.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/split_test.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform_append.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform_delete.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform_registry.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform_set.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform_set_test.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/transform_target.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/value_tpl.go (100%) rename x-pack/filebeat/input/httpjson/{ => internal}/v2/value_tpl_test.go (100%) diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 56417f1d800..4779b452f1d 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -13,7 +13,6 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" - httpjsonv2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/v2" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" ) @@ -30,7 +29,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 cloudfoundry.Plugin(), http_endpoint.Plugin(), httpjson.Plugin(log, store), - httpjsonv2.Plugin(log, store), o365audit.Plugin(log, store), s3.Plugin(), } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 4df5bc80c7d..18f978be023 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -15,7 +15,7 @@ import ( "github.com/hashicorp/go-retryablehttp" "go.uber.org/zap" - v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputv2 "github.com/elastic/beats/v7/filebeat/input/v2" cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/beat" @@ -24,6 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/useragent" "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/logp" + v2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/v2" "github.com/elastic/go-concert/ctxtool" "github.com/elastic/go-concert/timed" ) @@ -65,14 +66,15 @@ func (log *retryLogger) Warn(format string, args ...interface{}) { log.log.Warnf(format, args...) } -func Plugin(log *logp.Logger, store cursor.StateStore) v2.Plugin { +func Plugin(log *logp.Logger, store cursor.StateStore) inputv2.Plugin { sim := stateless.NewInputManager(statelessConfigure) - return v2.Plugin{ + return inputv2.Plugin{ Name: inputName, Stability: feature.Beta, - Deprecated: true, + Deprecated: false, Manager: inputManager{ - stateless: &sim, + v2inputManager: v2.NewInputManager(), + stateless: &sim, cursor: &cursor.InputManager{ Logger: log, StateStore: store, @@ -117,7 +119,7 @@ func test(url *url.URL) error { } func run( - ctx v2.Context, + ctx inputv2.Context, config config, tlsConfig *tlscommon.TLSConfig, publisher cursor.Publisher, diff --git a/x-pack/filebeat/input/httpjson/input_manager.go b/x-pack/filebeat/input/httpjson/input_manager.go index 8d7e6070786..31560839b0e 100644 --- a/x-pack/filebeat/input/httpjson/input_manager.go +++ b/x-pack/filebeat/input/httpjson/input_manager.go @@ -9,10 +9,11 @@ import ( "github.com/elastic/go-concert/unison" - v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputv2 "github.com/elastic/beats/v7/filebeat/input/v2" cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/common" + v2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/v2" ) // inputManager wraps one stateless input manager @@ -21,21 +22,29 @@ import ( type inputManager struct { stateless *stateless.InputManager cursor *cursor.InputManager + + v2inputManager v2.InputManager } -var _ v2.InputManager = inputManager{} +var _ inputv2.InputManager = inputManager{} // Init initializes both wrapped input managers. -func (m inputManager) Init(grp unison.Group, mode v2.Mode) error { +func (m inputManager) Init(grp unison.Group, mode inputv2.Mode) error { return multierr.Append( - m.stateless.Init(grp, mode), - m.cursor.Init(grp, mode), + multierr.Append( + m.stateless.Init(grp, mode), + m.cursor.Init(grp, mode), + ), + m.v2inputManager.Init(grp, mode), ) } // Create creates a cursor input manager if the config has a date cursor set up, // otherwise it creates a stateless input manager. -func (m inputManager) Create(cfg *common.Config) (v2.Input, error) { +func (m inputManager) Create(cfg *common.Config) (inputv2.Input, error) { + if b, _ := cfg.Bool("is_v2", -1); b { + return m.v2inputManager.Create(cfg) + } config := newDefaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err diff --git a/x-pack/filebeat/input/httpjson/v2/config.go b/x-pack/filebeat/input/httpjson/internal/v2/config.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/config.go rename to x-pack/filebeat/input/httpjson/internal/v2/config.go diff --git a/x-pack/filebeat/input/httpjson/v2/config_auth.go b/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/config_auth.go rename to x-pack/filebeat/input/httpjson/internal/v2/config_auth.go diff --git a/x-pack/filebeat/input/httpjson/v2/config_oauth_test.go b/x-pack/filebeat/input/httpjson/internal/v2/config_oauth_test.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/config_oauth_test.go rename to x-pack/filebeat/input/httpjson/internal/v2/config_oauth_test.go diff --git a/x-pack/filebeat/input/httpjson/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/config_request.go rename to x-pack/filebeat/input/httpjson/internal/v2/config_request.go diff --git a/x-pack/filebeat/input/httpjson/v2/config_response.go b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/config_response.go rename to x-pack/filebeat/input/httpjson/internal/v2/config_response.go diff --git a/x-pack/filebeat/input/httpjson/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go similarity index 91% rename from x-pack/filebeat/input/httpjson/v2/input.go rename to x-pack/filebeat/input/httpjson/internal/v2/input.go index cd93ee3cec8..37a299a6b4c 100644 --- a/x-pack/filebeat/input/httpjson/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -18,12 +18,10 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" - stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" "github.com/elastic/beats/v7/libbeat/common/useragent" - "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/go-concert/ctxtool" "github.com/elastic/go-concert/timed" @@ -66,23 +64,6 @@ func (log *retryLogger) Warn(format string, args ...interface{}) { log.log.Warnf(format, args...) } -func Plugin(log *logp.Logger, store cursor.StateStore) v2.Plugin { - sim := stateless.NewInputManager(statelessConfigure) - - registerRequestTransforms() - registerResponseTransforms() - registerPaginationTransforms() - - return v2.Plugin{ - Name: inputName, - Stability: feature.Beta, - Deprecated: false, - Manager: inputManager{ - stateless: &sim, - }, - } -} - func newTLSConfig(config config) (*tlscommon.TLSConfig, error) { if err := config.Validate(); err != nil { return nil, err diff --git a/x-pack/filebeat/input/httpjson/v2/input_manager.go b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go similarity index 71% rename from x-pack/filebeat/input/httpjson/v2/input_manager.go rename to x-pack/filebeat/input/httpjson/internal/v2/input_manager.go index 473e33fa7c1..fa830e6b5e2 100644 --- a/x-pack/filebeat/input/httpjson/v2/input_manager.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go @@ -15,20 +15,30 @@ import ( // inputManager wraps one stateless input manager // and one cursor input manager. It will create one or the other // based on the config that is passed. -type inputManager struct { +type InputManager struct { stateless *stateless.InputManager } -var _ v2.InputManager = inputManager{} +var _ v2.InputManager = InputManager{} + +func NewInputManager() InputManager { + sim := stateless.NewInputManager(statelessConfigure) + return InputManager{ + stateless: &sim, + } +} // Init initializes both wrapped input managers. -func (m inputManager) Init(grp unison.Group, mode v2.Mode) error { +func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { + registerRequestTransforms() + registerResponseTransforms() + registerPaginationTransforms() return m.stateless.Init(grp, mode) // multierr.Append() } // Create creates a cursor input manager if the config has a date cursor set up, // otherwise it creates a stateless input manager. -func (m inputManager) Create(cfg *common.Config) (v2.Input, error) { +func (m InputManager) Create(cfg *common.Config) (v2.Input, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err diff --git a/x-pack/filebeat/input/httpjson/v2/input_stateless.go b/x-pack/filebeat/input/httpjson/internal/v2/input_stateless.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/input_stateless.go rename to x-pack/filebeat/input/httpjson/internal/v2/input_stateless.go diff --git a/x-pack/filebeat/input/httpjson/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/pagination.go rename to x-pack/filebeat/input/httpjson/internal/v2/pagination.go diff --git a/x-pack/filebeat/input/httpjson/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/request.go rename to x-pack/filebeat/input/httpjson/internal/v2/request.go diff --git a/x-pack/filebeat/input/httpjson/v2/response.go b/x-pack/filebeat/input/httpjson/internal/v2/response.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/response.go rename to x-pack/filebeat/input/httpjson/internal/v2/response.go diff --git a/x-pack/filebeat/input/httpjson/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/split.go rename to x-pack/filebeat/input/httpjson/internal/v2/split.go diff --git a/x-pack/filebeat/input/httpjson/v2/split_test.go b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/split_test.go rename to x-pack/filebeat/input/httpjson/internal/v2/split_test.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform.go b/x-pack/filebeat/input/httpjson/internal/v2/transform.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform_append.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform_append.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_delete.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform_delete.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_registry.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform_registry.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_set.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform_set.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform_set.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform_set_test.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go diff --git a/x-pack/filebeat/input/httpjson/v2/transform_target.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/transform_target.go rename to x-pack/filebeat/input/httpjson/internal/v2/transform_target.go diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/value_tpl.go rename to x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go diff --git a/x-pack/filebeat/input/httpjson/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go similarity index 100% rename from x-pack/filebeat/input/httpjson/v2/value_tpl_test.go rename to x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go From c410215bf0a2d67c71fcc455403dd2eb6a31ea9f Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 6 Nov 2020 13:53:19 +0100 Subject: [PATCH 20/35] Add cursor --- x-pack/filebeat/input/httpjson/input.go | 2 +- .../input/httpjson/internal/v2/config.go | 6 ++ .../httpjson/internal/v2/config_request.go | 2 +- .../httpjson/internal/v2/config_response.go | 8 +- .../input/httpjson/internal/v2/cursor.go | 63 +++++++++++ .../input/httpjson/internal/v2/input.go | 12 +-- .../httpjson/internal/v2/input_cursor.go | 67 ++++++++++++ .../httpjson/internal/v2/input_manager.go | 16 ++- .../input/httpjson/internal/v2/pagination.go | 27 +++-- .../input/httpjson/internal/v2/request.go | 28 +++-- .../input/httpjson/internal/v2/response.go | 33 +++--- .../input/httpjson/internal/v2/split.go | 49 ++++----- .../input/httpjson/internal/v2/split_test.go | 101 ++++++++---------- .../input/httpjson/internal/v2/transform.go | 47 +++++--- .../httpjson/internal/v2/transform_append.go | 19 ++-- .../httpjson/internal/v2/transform_delete.go | 7 +- .../internal/v2/transform_registry.go | 2 +- .../httpjson/internal/v2/transform_set.go | 26 +++-- .../internal/v2/transform_set_test.go | 7 +- .../input/httpjson/internal/v2/value_tpl.go | 23 ++-- .../httpjson/internal/v2/value_tpl_test.go | 12 ++- 21 files changed, 365 insertions(+), 192 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/cursor.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 18f978be023..3d476b143ef 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -73,7 +73,7 @@ func Plugin(log *logp.Logger, store cursor.StateStore) inputv2.Plugin { Stability: feature.Beta, Deprecated: false, Manager: inputManager{ - v2inputManager: v2.NewInputManager(), + v2inputManager: v2.NewInputManager(log, store), stateless: &sim, cursor: &cursor.InputManager{ Logger: log, diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config.go b/x-pack/filebeat/input/httpjson/internal/v2/config.go index 35056e44eca..893d17ccfad 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config.go @@ -14,6 +14,12 @@ type config struct { Auth *authConfig `config:"auth"` Request *requestConfig `config:"request" validate:"required"` Response *responseConfig `config:"response"` + Cursor cursorConfig `config:"cursor"` +} + +type cursorConfig map[string]struct { + Value *valueTpl `config:"value"` + Default string `config:"default"` } func (c config) Validate() error { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go index 855edc26977..5749e442157 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go @@ -121,7 +121,7 @@ func (c *requestConfig) Validate() error { return errors.New("timeout must be greater than 0") } - if _, err := newBasicTransformsFromConfig(c.Transforms, requestNamespace); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, requestNamespace, nil); err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go index b1b81081200..55b45650a2d 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go @@ -30,17 +30,17 @@ type splitConfig struct { } func (c *responseConfig) Validate() error { - if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace, nil); err != nil { return err } - if _, err := newBasicTransformsFromConfig(c.Transforms, paginationNamespace); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, paginationNamespace, nil); err != nil { return err } return nil } func (c *splitConfig) Validate() error { - if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace); err != nil { + if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace, nil); err != nil { return err } @@ -55,7 +55,7 @@ func (c *splitConfig) Validate() error { return fmt.Errorf("invalid split type: %s", c.Type) } - if _, err := newSplitResponse(c); err != nil { + if _, err := newSplitResponse(c, nil); err != nil { return err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go b/x-pack/filebeat/input/httpjson/internal/v2/cursor.go new file mode 100644 index 00000000000..ca555b80777 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/cursor.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type cursor struct { + log *logp.Logger + + cfg cursorConfig + + state common.MapStr +} + +func newCursor(cfg cursorConfig, log *logp.Logger) *cursor { + return &cursor{cfg: cfg, log: log} +} + +func (c *cursor) load(cursor *inputcursor.Cursor) { + if c == nil || cursor == nil || cursor.IsNew() { + return + } + + if c.state == nil { + c.state = common.MapStr{} + } + + if err := cursor.Unpack(&c.state); err != nil { + c.log.Errorf("Reset cursor state. Failed to read from registry: %v", err) + return + } + + c.log.Debugf("cursor loaded: %v", c.state) +} + +func (c *cursor) update(trCtx transformContext) { + if c.cfg == nil { + return + } + + if c.state == nil { + c.state = common.MapStr{} + } + + for k, cfg := range c.cfg { + v := cfg.Value.Execute(trCtx, emptyTransformable(), cfg.Default, c.log) + _, _ = c.state.Put(k, v) + c.log.Debugf("cursor.%s stored with %s", k, v) + } +} + +func (c *cursor) clone() common.MapStr { + if c == nil || c.state == nil { + return common.MapStr{} + } + return c.state.Clone() +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go index 37a299a6b4c..3c1ce8c9dda 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -17,7 +17,7 @@ import ( "go.uber.org/zap" v2 "github.com/elastic/beats/v7/filebeat/input/v2" - cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" @@ -101,8 +101,8 @@ func run( ctx v2.Context, config config, tlsConfig *tlscommon.TLSConfig, - publisher cursor.Publisher, - cursor *cursor.Cursor, + publisher inputcursor.Publisher, + cursor *inputcursor.Cursor, ) error { log := ctx.Logger.With("url", config.Request.URL) @@ -115,12 +115,12 @@ func run( requestFactory := newRequestFactory(config.Request, config.Auth, log) pagination := newPagination(config, httpClient, log) - responseProcessor := newResponseProcessor(config.Response, pagination) + responseProcessor := newResponseProcessor(config.Response, pagination, log) requester := newRequester(httpClient, requestFactory, responseProcessor, log) - // loadContextFromCursor trCtx := emptyTransformContext() - // + trCtx.cursor = newCursor(config.Cursor, log) + trCtx.cursor.load(cursor) err = timed.Periodic(stdCtx, config.Interval, func() error { log.Info("Process another repeated request.") diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go b/x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go new file mode 100644 index 00000000000..537e67762df --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_cursor.go @@ -0,0 +1,67 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" +) + +type cursorInput struct{} + +func (cursorInput) Name() string { + return "httpjson-cursor" +} + +type source struct { + config config + tlsConfig *tlscommon.TLSConfig +} + +func (src source) Name() string { + return src.config.Request.URL.String() +} + +func cursorConfigure(cfg *common.Config) ([]inputcursor.Source, inputcursor.Input, error) { + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + return nil, nil, err + } + return newCursorInput(conf) +} + +func newCursorInput(config config) ([]inputcursor.Source, inputcursor.Input, error) { + tlsConfig, err := newTLSConfig(config) + if err != nil { + return nil, nil, err + } + // we only allow one url per config, if we wanted to allow more than one + // each source should hold only one url + return []inputcursor.Source{ + &source{config: config, + tlsConfig: tlsConfig, + }, + }, + &cursorInput{}, + nil +} + +func (in *cursorInput) Test(src inputcursor.Source, _ v2.TestContext) error { + return test((src.(*source)).config.Request.URL.URL) +} + +// Run starts the input and blocks until it ends the execution. +// It will return on context cancellation, any other error will be retried. +func (in *cursorInput) Run( + ctx v2.Context, + src inputcursor.Source, + cursor inputcursor.Cursor, + publisher inputcursor.Publisher, +) error { + s := src.(*source) + return run(ctx, s.config, s.tlsConfig, publisher, &cursor) +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go index fa830e6b5e2..729ff417f46 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go @@ -8,8 +8,10 @@ import ( "github.com/elastic/go-concert/unison" v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) // inputManager wraps one stateless input manager @@ -17,14 +19,21 @@ import ( // based on the config that is passed. type InputManager struct { stateless *stateless.InputManager + cursor *inputcursor.InputManager } var _ v2.InputManager = InputManager{} -func NewInputManager() InputManager { +func NewInputManager(log *logp.Logger, store inputcursor.StateStore) InputManager { sim := stateless.NewInputManager(statelessConfigure) return InputManager{ stateless: &sim, + cursor: &inputcursor.InputManager{ + Logger: log, + StateStore: store, + Type: inputName, + Configure: cursorConfigure, + }, } } @@ -43,5 +52,8 @@ func (m InputManager) Create(cfg *common.Config) (v2.Input, error) { if err := cfg.Unpack(&config); err != nil { return nil, err } - return m.stateless.Create(cfg) + if len(config.Cursor) == 0 { + return m.stateless.Create(cfg) + } + return m.cursor.Create(cfg) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index 28358224251..c66b55c0dae 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -25,18 +25,19 @@ func registerPaginationTransforms() { } type pagination struct { + log *logp.Logger httpClient *http.Client requestFactory *requestFactory } func newPagination(config config, httpClient *http.Client, log *logp.Logger) *pagination { - pagination := &pagination{httpClient: httpClient} - if config.Response == nil { + pagination := &pagination{httpClient: httpClient, log: log} + if config.Response == nil || len(config.Response.Pagination) == 0 { return pagination } - rts, _ := newBasicTransformsFromConfig(config.Request.Transforms, requestNamespace) - pts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace) + rts, _ := newBasicTransformsFromConfig(config.Request.Transforms, requestNamespace, log) + pts, _ := newBasicTransformsFromConfig(config.Response.Pagination, paginationNamespace, log) requestFactory := newPaginationRequestFactory( config.Request.Method, *config.Request.URL.URL, @@ -73,6 +74,7 @@ type pageIterator struct { resp *http.Response isFirst bool + done bool } func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformContext, resp *http.Response) *pageIterator { @@ -86,16 +88,21 @@ func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformCont } func (iter *pageIterator) next() (*transformable, bool, error) { - if iter == nil || iter.resp == nil { + if iter == nil || iter.resp == nil || iter.done { return nil, false, nil } if iter.isFirst { + iter.pagination.log.Debug("first page requested") iter.isFirst = false tr, err := iter.getPage() if err != nil { return nil, false, err } + if iter.pagination.requestFactory == nil { + iter.pagination.log.Debug("last page") + iter.done = true + } return tr, true, nil } @@ -104,6 +111,8 @@ func (iter *pageIterator) next() (*transformable, bool, error) { if err == errNewURLValueNotSet { // if this error happens here it means the transform used to pick the new url.value // did not find any new value and we can stop paginating without error + iter.pagination.log.Debug("last page") + iter.done = true return nil, false, nil } return nil, false, err @@ -127,6 +136,8 @@ func (iter *pageIterator) next() (*transformable, bool, error) { } if len(tr.body) == 0 { + iter.pagination.log.Debug("finished pagination because there is no body") + iter.done = true return nil, false, nil } @@ -150,11 +161,5 @@ func (iter *pageIterator) getPage() (*transformable, error) { } } - iter.trCtx.lastResponse = &transformable{ - body: tr.body.Clone(), - header: tr.header.Clone(), - url: tr.url, - } - return tr, nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index beefbba683d..f19998cb664 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -13,7 +13,7 @@ import ( "net/http" "net/url" - cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -57,7 +57,7 @@ type requestFactory struct { func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { // config validation already checked for errors here - ts, _ := newBasicTransformsFromConfig(config.Transforms, requestNamespace) + ts, _ := newBasicTransformsFromConfig(config.Transforms, requestNamespace, log) rf := &requestFactory{ url: *config.URL.URL, method: config.Method, @@ -132,7 +132,7 @@ func newRequester( } } -func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, publisher cursor.Publisher) error { +func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, publisher inputcursor.Publisher) error { req, err := r.requestFactory.newHTTPRequest(stdCtx, trCtx) if err != nil { return fmt.Errorf("failed to create http request: %w", err) @@ -154,15 +154,29 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu return err } - for maybeEvent := range eventsCh { - if maybeEvent.failed() { - r.log.Errorf("error processing response: %v", maybeEvent) + var n int + for maybeMsg := range eventsCh { + if maybeMsg.failed() { + r.log.Errorf("error processing response: %v", maybeMsg) continue } - if err := publisher.Publish(maybeEvent.event, trCtx.cursor.Clone()); err != nil { + + event, err := makeEvent(maybeMsg.msg) + if err != nil { + r.log.Errorf("error creating event: %v", maybeMsg) + continue + } + + if err := publisher.Publish(event, trCtx.cursor.clone()); err != nil { r.log.Errorf("error publishing event: %v", err) + continue } + + *trCtx.lastEvent = maybeMsg.msg + trCtx.cursor.update(trCtx) + n += 1 } + r.log.Infof("request finished: %d events published", n) return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/response.go b/x-pack/filebeat/input/httpjson/internal/v2/response.go index 7978263107a..c98366ef434 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/response.go @@ -6,8 +6,9 @@ package v2 import ( "context" - "fmt" "net/http" + + "github.com/elastic/beats/v7/libbeat/logp" ) const responseNamespace = "response" @@ -19,30 +20,32 @@ func registerResponseTransforms() { } type responseProcessor struct { + log *logp.Logger transforms []basicTransform split *split pagination *pagination } -func newResponseProcessor(config *responseConfig, pagination *pagination) *responseProcessor { +func newResponseProcessor(config *responseConfig, pagination *pagination, log *logp.Logger) *responseProcessor { rp := &responseProcessor{ pagination: pagination, + log: log, } if config == nil { return rp } - ts, _ := newBasicTransformsFromConfig(config.Transforms, responseNamespace) + ts, _ := newBasicTransformsFromConfig(config.Transforms, responseNamespace, log) rp.transforms = ts - split, _ := newSplitResponse(config.Split) + split, _ := newSplitResponse(config.Split, log) rp.split = split return rp } -func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx transformContext, resp *http.Response) (<-chan maybeEvent, error) { - ch := make(chan maybeEvent) +func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx transformContext, resp *http.Response) (<-chan maybeMsg, error) { + ch := make(chan maybeMsg) go func() { defer close(ch) @@ -51,7 +54,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans for { page, hasNext, err := iter.next() if err != nil { - ch <- maybeEvent{err: err} + ch <- maybeMsg{err: err} return } @@ -59,22 +62,20 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans return } + *trCtx.lastResponse = *page.clone() + + rp.log.Debugf("last received page: %v", trCtx.lastResponse) + for _, t := range rp.transforms { page, err = t.run(trCtx, page) if err != nil { - fmt.Println("=== 2") - ch <- maybeEvent{err: err} + ch <- maybeMsg{err: err} return } } if rp.split == nil { - event, err := makeEvent(page.body) - if err != nil { - ch <- maybeEvent{err: err} - return - } - ch <- maybeEvent{event: event} + ch <- maybeMsg{msg: page.body} continue } @@ -84,7 +85,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans return } - ch <- maybeEvent{err: err} + ch <- maybeMsg{err: err} return } } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go index b0f699876a9..8748f9e4efa 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) var errEmtpyField = errors.New("the requested field is emtpy") @@ -22,12 +23,12 @@ type split struct { keyField string } -func newSplitResponse(cfg *splitConfig) (*split, error) { +func newSplitResponse(cfg *splitConfig, log *logp.Logger) (*split, error) { if cfg == nil { return nil, nil } - split, err := newSplit(cfg) + split, err := newSplit(cfg, log) if err != nil { return nil, err } @@ -39,20 +40,20 @@ func newSplitResponse(cfg *splitConfig) (*split, error) { return split, nil } -func newSplit(c *splitConfig) (*split, error) { +func newSplit(c *splitConfig, log *logp.Logger) (*split, error) { ti, err := getTargetInfo(c.Target) if err != nil { return nil, err } - ts, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace) + ts, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace, log) if err != nil { return nil, err } var s *split if c.Split != nil { - s, err = newSplitResponse(c.Split) + s, err = newSplitResponse(c.Split, log) if err != nil { return nil, err } @@ -68,7 +69,7 @@ func newSplit(c *splitConfig) (*split, error) { }, nil } -func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEvent) error { +func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMsg) error { respCpy := resp.clone() var err error for _, t := range s.transforms { @@ -95,12 +96,7 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEv } for _, a := range arr { - m, ok := toMapStr(a) - if !ok { - return errors.New("split can only be applied on object lists") - } - - if err := s.sendEvent(ctx, respCpy, m, ch); err != nil { + if err := s.sendEvent(ctx, respCpy, "", a, ch); err != nil { return err } } @@ -121,14 +117,7 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeEv } for k, v := range ms { - m, ok := toMapStr(v) - if !ok { - return errors.New("split can only be applied on object lists") - } - if s.keyField != "" { - _, _ = m.Put(s.keyField, k) - } - if err := s.sendEvent(ctx, respCpy, m, ch); err != nil { + if err := s.sendEvent(ctx, respCpy, k, v, ch); err != nil { return err } } @@ -152,7 +141,16 @@ func toMapStr(v interface{}) (common.MapStr, bool) { return m, true } -func (s *split) sendEvent(ctx transformContext, resp *transformable, m common.MapStr, ch chan<- maybeEvent) error { +func (s *split) sendEvent(ctx transformContext, resp *transformable, key string, val interface{}, ch chan<- maybeMsg) error { + m, ok := toMapStr(val) + if !ok { + return errors.New("split can only be applied on object lists") + } + + if s.keyField != "" && key != "" { + _, _ = m.Put(s.keyField, key) + } + if s.keepParent { _, _ = resp.body.Put(s.targetInfo.Name, m) } else { @@ -163,14 +161,7 @@ func (s *split) sendEvent(ctx transformContext, resp *transformable, m common.Ma return s.split.run(ctx, resp, ch) } - event, err := makeEvent(resp.body) - if err != nil { - return err - } - - ch <- maybeEvent{event: event} - - *ctx.lastEvent = event + ch <- maybeMsg{msg: resp.body.Clone()} return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go index 74d262cb1eb..0e42ba91f1a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go @@ -1,10 +1,16 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( "testing" - "github.com/elastic/beats/v7/libbeat/common" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) func TestSplit(t *testing.T) { @@ -13,7 +19,7 @@ func TestSplit(t *testing.T) { config *splitConfig ctx transformContext resp *transformable - expectedMessages []string + expectedMessages []common.MapStr expectedErr error }{ { @@ -58,43 +64,27 @@ func TestSplit(t *testing.T) { }, }, }, - expectedMessages: []string{ - `{ - "this": "is kept", - "alerts": { - "this_is": "also kept", - "entities": { - "something": "something" - } - } - }`, - `{ - "this": "is kept", - "alerts": { - "this_is": "also kept", - "entities": { - "else": "else" - } - } - }`, - `{ - "this": "is kept", - "alerts": { - "this_is": "also kept 2", - "entities": { - "something": "something 2" - } - } - }`, - `{ - "this": "is kept", - "alerts": { - "this_is": "also kept 2", - "entities": { - "else": "else 2" - } - } - }`, + expectedMessages: []common.MapStr{ + { + "this": "is kept", + "alerts.this_is": "also kept", + "alerts.entities.something": "something", + }, + { + "this": "is kept", + "alerts.this_is": "also kept", + "alerts.entities.else": "else", + }, + { + "this": "is kept", + "alerts.this_is": "also kept 2", + "alerts.entities.something": "something 2", + }, + { + "this": "is kept", + "alerts.this_is": "also kept 2", + "alerts.entities.else": "else 2", + }, }, expectedErr: nil, }, @@ -135,21 +125,17 @@ func TestSplit(t *testing.T) { }, }, }, - expectedMessages: []string{ - `{ - "this_is": "kept", - "entities": { - "id": "id1", - "something": "else" - } - }`, - `{ - "this_is": "also kept", - "entities": { - "id": "id2", - "something": "else 2" - } - }`, + expectedMessages: []common.MapStr{ + { + "this_is": "kept", + "entities.id": "id1", + "entities.something": "else", + }, + { + "this_is": "also kept", + "entities.id": "id2", + "entities.something": "else 2", + }, }, expectedErr: nil, }, @@ -158,8 +144,8 @@ func TestSplit(t *testing.T) { for _, tc := range cases { tc := tc t.Run(tc.name, func(t *testing.T) { - ch := make(chan maybeEvent, len(tc.expectedMessages)) - split, err := newSplitResponse(tc.config) + ch := make(chan maybeMsg, len(tc.expectedMessages)) + split, err := newSplitResponse(tc.config, logp.NewLogger("")) assert.NoError(t, err) err = split.run(tc.ctx, tc.resp, ch) if tc.expectedErr == nil { @@ -172,8 +158,7 @@ func TestSplit(t *testing.T) { for _, msg := range tc.expectedMessages { e := <-ch assert.NoError(t, e.err) - got := e.event.Fields["message"].(string) - assert.JSONEq(t, msg, got) + assert.Equal(t, msg.Flatten(), e.msg.Flatten()) } }) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform.go b/x-pack/filebeat/input/httpjson/internal/v2/transform.go index f1fa5397270..e0b8a86a114 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform.go @@ -12,8 +12,8 @@ import ( "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) const logName = "httpjson.transforms" @@ -23,16 +23,16 @@ type transformsConfig []*common.Config type transforms []transform type transformContext struct { - cursor common.MapStr - lastEvent *beat.Event + cursor *cursor + lastEvent *common.MapStr lastResponse *transformable } func emptyTransformContext() transformContext { return transformContext{ - cursor: make(common.MapStr), + cursor: &cursor{}, + lastEvent: &common.MapStr{}, lastResponse: emptyTransformable(), - lastEvent: &beat.Event{}, } } @@ -43,10 +43,23 @@ type transformable struct { } func (t *transformable) clone() *transformable { + if t == nil { + return emptyTransformable() + } return &transformable{ - body: t.body.Clone(), - header: t.header.Clone(), - url: t.url, + body: func() common.MapStr { + if t.body == nil { + return common.MapStr{} + } + return t.body.Clone() + }(), + header: func() http.Header { + if t.header == nil { + return http.Header{} + } + return t.header.Clone() + }(), + url: t.url, } } @@ -66,17 +79,17 @@ type basicTransform interface { run(transformContext, *transformable) (*transformable, error) } -type maybeEvent struct { - err error - event beat.Event +type maybeMsg struct { + err error + msg common.MapStr } -func (e maybeEvent) failed() bool { return e.err != nil } +func (e maybeMsg) failed() bool { return e.err != nil } -func (e maybeEvent) Error() string { return e.err.Error() } +func (e maybeMsg) Error() string { return e.err.Error() } // newTransformsFromConfig creates a list of transforms from a list of free user configurations. -func newTransformsFromConfig(config transformsConfig, namespace string) (transforms, error) { +func newTransformsFromConfig(config transformsConfig, namespace string, log *logp.Logger) (transforms, error) { var trans transforms for _, tfConfig := range config { @@ -100,7 +113,7 @@ func newTransformsFromConfig(config transformsConfig, namespace string) (transfo } cfg.PrintDebugf("Configure transform '%v' with:", actionName) - transform, err := constructor(cfg) + transform, err := constructor(cfg, log) if err != nil { return nil, err } @@ -111,8 +124,8 @@ func newTransformsFromConfig(config transformsConfig, namespace string) (transfo return trans, nil } -func newBasicTransformsFromConfig(config transformsConfig, namespace string) ([]basicTransform, error) { - ts, err := newTransformsFromConfig(config, namespace) +func newBasicTransformsFromConfig(config transformsConfig, namespace string, log *logp.Logger) ([]basicTransform, error) { + ts, err := newTransformsFromConfig(config, namespace, log) if err != nil { return nil, err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go index 650557b2d1c..ebe9e821b74 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) const appendName = "append" @@ -21,6 +22,7 @@ type appendConfig struct { } type appendt struct { + log *logp.Logger targetInfo targetInfo value *valueTpl defaultValue string @@ -30,8 +32,8 @@ type appendt struct { func (appendt) transformName() string { return appendName } -func newAppendRequest(cfg *common.Config) (transform, error) { - append, err := newAppend(cfg) +func newAppendRequest(cfg *common.Config, log *logp.Logger) (transform, error) { + append, err := newAppend(cfg, log) if err != nil { return nil, err } @@ -50,8 +52,8 @@ func newAppendRequest(cfg *common.Config) (transform, error) { return &append, nil } -func newAppendResponse(cfg *common.Config) (transform, error) { - append, err := newAppend(cfg) +func newAppendResponse(cfg *common.Config, log *logp.Logger) (transform, error) { + append, err := newAppend(cfg, log) if err != nil { return nil, err } @@ -66,8 +68,8 @@ func newAppendResponse(cfg *common.Config) (transform, error) { return &append, nil } -func newAppendPagination(cfg *common.Config) (transform, error) { - append, err := newAppend(cfg) +func newAppendPagination(cfg *common.Config, log *logp.Logger) (transform, error) { + append, err := newAppend(cfg, log) if err != nil { return nil, err } @@ -86,7 +88,7 @@ func newAppendPagination(cfg *common.Config) (transform, error) { return &append, nil } -func newAppend(cfg *common.Config) (appendt, error) { +func newAppend(cfg *common.Config, log *logp.Logger) (appendt, error) { c := &appendConfig{} if err := cfg.Unpack(c); err != nil { return appendt{}, errors.Wrap(err, "fail to unpack the append configuration") @@ -98,6 +100,7 @@ func newAppend(cfg *common.Config) (appendt, error) { } return appendt{ + log: log, targetInfo: ti, value: c.Value, defaultValue: c.Default, @@ -105,7 +108,7 @@ func newAppend(cfg *common.Config) (appendt, error) { } func (append *appendt) run(ctx transformContext, transformable *transformable) (*transformable, error) { - value := append.value.Execute(ctx, transformable, append.defaultValue) + value := append.value.Execute(ctx, transformable, append.defaultValue, append.log) if err := append.runFunc(ctx, transformable, append.targetInfo.Name, value); err != nil { return nil, err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go index 085bbb7a5a8..a00715ae4c9 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) const deleteName = "delete" @@ -26,7 +27,7 @@ type delete struct { func (delete) transformName() string { return deleteName } -func newDeleteRequest(cfg *common.Config) (transform, error) { +func newDeleteRequest(cfg *common.Config, _ *logp.Logger) (transform, error) { delete, err := newDelete(cfg) if err != nil { return nil, err @@ -46,7 +47,7 @@ func newDeleteRequest(cfg *common.Config) (transform, error) { return &delete, nil } -func newDeleteResponse(cfg *common.Config) (transform, error) { +func newDeleteResponse(cfg *common.Config, _ *logp.Logger) (transform, error) { delete, err := newDelete(cfg) if err != nil { return nil, err @@ -62,7 +63,7 @@ func newDeleteResponse(cfg *common.Config) (transform, error) { return &delete, nil } -func newDeletePagination(cfg *common.Config) (transform, error) { +func newDeletePagination(cfg *common.Config, _ *logp.Logger) (transform, error) { delete, err := newDelete(cfg) if err != nil { return nil, err diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go index e585fc42e04..f0073f29277 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_registry.go @@ -13,7 +13,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -type constructor func(config *common.Config) (transform, error) +type constructor func(config *common.Config, log *logp.Logger) (transform, error) var registeredTransforms = newRegistry() diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go index e9ce6fe1e4e..4da04799db5 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go @@ -1,11 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( "fmt" "net/url" - "github.com/elastic/beats/v7/libbeat/common" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) var errNewURLValueNotSet = errors.New("the new url.value was not set") @@ -19,6 +25,7 @@ type setConfig struct { } type set struct { + log *logp.Logger targetInfo targetInfo value *valueTpl defaultValue string @@ -28,8 +35,8 @@ type set struct { func (set) transformName() string { return setName } -func newSetRequest(cfg *common.Config) (transform, error) { - set, err := newSet(cfg) +func newSetRequest(cfg *common.Config, log *logp.Logger) (transform, error) { + set, err := newSet(cfg, log) if err != nil { return nil, err } @@ -48,8 +55,8 @@ func newSetRequest(cfg *common.Config) (transform, error) { return &set, nil } -func newSetResponse(cfg *common.Config) (transform, error) { - set, err := newSet(cfg) +func newSetResponse(cfg *common.Config, log *logp.Logger) (transform, error) { + set, err := newSet(cfg, log) if err != nil { return nil, err } @@ -64,8 +71,8 @@ func newSetResponse(cfg *common.Config) (transform, error) { return &set, nil } -func newSetPagination(cfg *common.Config) (transform, error) { - set, err := newSet(cfg) +func newSetPagination(cfg *common.Config, log *logp.Logger) (transform, error) { + set, err := newSet(cfg, log) if err != nil { return nil, err } @@ -86,7 +93,7 @@ func newSetPagination(cfg *common.Config) (transform, error) { return &set, nil } -func newSet(cfg *common.Config) (set, error) { +func newSet(cfg *common.Config, log *logp.Logger) (set, error) { c := &setConfig{} if err := cfg.Unpack(c); err != nil { return set{}, errors.Wrap(err, "fail to unpack the set configuration") @@ -98,6 +105,7 @@ func newSet(cfg *common.Config) (set, error) { } return set{ + log: log, targetInfo: ti, value: c.Value, defaultValue: c.Default, @@ -105,7 +113,7 @@ func newSet(cfg *common.Config) (set, error) { } func (set *set) run(ctx transformContext, transformable *transformable) (*transformable, error) { - value := set.value.Execute(ctx, transformable, set.defaultValue) + value := set.value.Execute(ctx, transformable, set.defaultValue, set.log) if err := set.runFunc(ctx, transformable, set.targetInfo.Name, value); err != nil { return nil, err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go index c94f9df5250..40d207c3a20 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( @@ -5,8 +9,9 @@ import ( "net/url" "testing" - "github.com/elastic/beats/v7/libbeat/common" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" ) func TestSetFunctions(t *testing.T) { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index e427583c471..fe891467553 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -10,8 +10,8 @@ import ( "text/template" "time" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) type valueTpl struct { @@ -41,9 +41,11 @@ func (t *valueTpl) Unpack(in string) error { return nil } -func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal string) (val string) { +func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal string, log *logp.Logger) (val string) { defer func() { if r := recover(); r != nil { + err := r.(error) + log.Infof("template execution: %v", err) val = defaultVal } }() @@ -55,14 +57,15 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal _, _ = data.Put("body", tr.body.Clone()) _, _ = data.Put("url.value", tr.url.String()) _, _ = data.Put("url.params", tr.url.Query()) - _, _ = data.Put("cursor", trCtx.cursor.Clone()) - _, _ = data.Put("last_event", cloneEvent(trCtx.lastEvent)) + _, _ = data.Put("cursor", trCtx.cursor.clone()) + _, _ = data.Put("last_event", trCtx.lastEvent.Clone()) _, _ = data.Put("last_response.body", trCtx.lastResponse.body.Clone()) _, _ = data.Put("last_response.header", trCtx.lastResponse.header.Clone()) _, _ = data.Put("last_response.url.value", trCtx.lastResponse.url.String()) _, _ = data.Put("last_response.url.params", trCtx.lastResponse.url.Query()) if err := t.Template.Execute(buf, data); err != nil { + log.Infof("template execution: %v", err) return defaultVal } @@ -73,18 +76,6 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal return val } -func cloneEvent(event *beat.Event) beat.Event { - if event == nil { - return beat.Event{} - } - return beat.Event{ - Timestamp: event.Timestamp, - Meta: event.Meta.Clone(), - Fields: event.Fields.Clone(), - TimeSeries: event.TimeSeries, - } -} - var ( predefinedLayouts = map[string]string{ "ANSIC": time.ANSIC, diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index cac77ec54ae..9a8b2e37be9 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package v2 import ( @@ -5,8 +9,10 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" ) func TestValueTpl(t *testing.T) { @@ -24,6 +30,7 @@ func TestValueTpl(t *testing.T) { name: "can render values from ctx", value: "{{.last_response.body.param}}", paramCtx: transformContext{ + lastEvent: &common.MapStr{}, lastResponse: newTestTransformable(common.MapStr{"param": 25}, nil, ""), }, paramTr: emptyTransformable(), @@ -152,6 +159,7 @@ func TestValueTpl(t *testing.T) { name: "func getRFC5988Link", value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, paramCtx: transformContext{ + lastEvent: &common.MapStr{}, lastResponse: newTestTransformable( nil, http.Header{"Link": []string{ @@ -210,7 +218,7 @@ func TestValueTpl(t *testing.T) { } tpl := &valueTpl{} assert.NoError(t, tpl.Unpack(tc.value)) - got := tpl.Execute(tc.paramCtx, tc.paramTr, tc.paramDefVal) + got := tpl.Execute(tc.paramCtx, tc.paramTr, tc.paramDefVal, logp.NewLogger("")) assert.Equal(t, tc.expected, got) }) } From 12857e1c6f690b35e7e57ba62f34165110df34e0 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 6 Nov 2020 16:54:06 +0100 Subject: [PATCH 21/35] Add rate limit --- .../httpjson/internal/v2/config_request.go | 8 - .../input/httpjson/internal/v2/input.go | 14 +- .../input/httpjson/internal/v2/pagination.go | 12 +- .../httpjson/internal/v2/rate_limiter.go | 140 ++++++++++++++++++ .../httpjson/internal/v2/rate_limiter_test.go | 87 +++++++++++ .../input/httpjson/internal/v2/request.go | 31 +++- 6 files changed, 263 insertions(+), 29 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go index 5749e442157..2978f42cc47 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go @@ -60,14 +60,6 @@ type rateLimitConfig struct { Remaining *valueTpl `config:"remaining"` } -func (c rateLimitConfig) Validate() error { - if c.Limit == nil || c.Reset == nil || c.Remaining == nil { - return errors.New("all rate_limit fields must have a value") - } - - return nil -} - type urlConfig struct { *url.URL } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go index 3c1ce8c9dda..11e6eb8be8a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -108,7 +108,7 @@ func run( stdCtx := ctxtool.FromCanceller(ctx.Cancelation) - httpClient, err := newHTTPClient(stdCtx, config, tlsConfig) + httpClient, err := newHTTPClient(stdCtx, config, tlsConfig, log) if err != nil { return err } @@ -141,7 +141,7 @@ func run( return nil } -func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig) (*http.Client, error) { +func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig, log *logp.Logger) (*httpClient, error) { timeout := config.Request.getTimeout() // Make retryable HTTP client @@ -165,11 +165,17 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC Backoff: retryablehttp.DefaultBackoff, } + limiter := newRateLimiterFromConfig(config.Request.RateLimit, log) + if config.Auth.OAuth2.isEnabled() { - return config.Auth.OAuth2.client(ctx, client.StandardClient()) + authClient, err := config.Auth.OAuth2.client(ctx, client.StandardClient()) + if err != nil { + return nil, err + } + return &httpClient{client: authClient, limiter: limiter}, nil } - return client.StandardClient(), nil + return &httpClient{client: client.StandardClient(), limiter: limiter}, nil } func checkRedirect(config *requestConfig) func(*http.Request, []*http.Request) error { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index c66b55c0dae..a49e1d3800c 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -7,7 +7,6 @@ package v2 import ( "context" "encoding/json" - "fmt" "io/ioutil" "net/http" "net/url" @@ -26,11 +25,11 @@ func registerPaginationTransforms() { type pagination struct { log *logp.Logger - httpClient *http.Client + httpClient *httpClient requestFactory *requestFactory } -func newPagination(config config, httpClient *http.Client, log *logp.Logger) *pagination { +func newPagination(config config, httpClient *httpClient, log *logp.Logger) *pagination { pagination := &pagination{httpClient: httpClient, log: log} if config.Response == nil || len(config.Response.Pagination) == 0 { return pagination @@ -118,16 +117,11 @@ func (iter *pageIterator) next() (*transformable, bool, error) { return nil, false, err } - resp, err := iter.pagination.httpClient.Do(httpReq) + resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) if err != nil { return nil, false, err } - if resp.StatusCode > 399 { - body, _ := ioutil.ReadAll(resp.Body) - return nil, false, fmt.Errorf("server responded with status code %d: %s", resp.StatusCode, string(body)) - } - iter.resp = resp tr, err := iter.getPage() diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go new file mode 100644 index 00000000000..2e8fb9b01d9 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go @@ -0,0 +1,140 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +type rateLimiter struct { + log *logp.Logger + + limit *valueTpl + reset *valueTpl + remaining *valueTpl +} + +func newRateLimiterFromConfig(config *rateLimitConfig, log *logp.Logger) *rateLimiter { + if config == nil { + return nil + } + + return &rateLimiter{ + log: log, + limit: config.Limit, + reset: config.Reset, + remaining: config.Remaining, + } +} + +func (r *rateLimiter) execute(ctx context.Context, f func() (*http.Response, error)) (*http.Response, error) { + for { + resp, err := f() + if err != nil { + return nil, err + } + + if err != nil { + return nil, fmt.Errorf("failed to read http.response.body: %w", err) + } + + if r == nil || resp.StatusCode == http.StatusOK { + return resp, nil + } + + if resp.StatusCode != http.StatusTooManyRequests { + return nil, fmt.Errorf("http request was unsuccessful with a status code %d", resp.StatusCode) + } + + if err := r.applyRateLimit(ctx, resp); err != nil { + return nil, err + } + } +} + +// applyRateLimit applies appropriate rate limit if specified in the HTTP Header of the response +func (r *rateLimiter) applyRateLimit(ctx context.Context, resp *http.Response) error { + epoch, err := r.getRateLimit(resp) + if err != nil { + return err + } + + t := time.Unix(epoch, 0) + w := time.Until(t) + if epoch == 0 || w <= 0 { + r.log.Debugf("Rate Limit: No need to apply rate limit.") + return nil + } + r.log.Debugf("Rate Limit: Wait until %v for the rate limit to reset.", t) + ticker := time.NewTicker(w) + defer ticker.Stop() + + select { + case <-ctx.Done(): + r.log.Info("Context done.") + return nil + case <-ticker.C: + r.log.Debug("Rate Limit: time is up.") + return nil + } +} + +// getRateLimit gets the rate limit value if specified in the response, +// and returns an int64 value in seconds since unix epoch for rate limit reset time. +// When there is a remaining rate limit quota, or when the rate limit reset time has expired, it +// returns 0 for the epoch value. +func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { + if r == nil { + return 0, nil + } + + if r.remaining == nil { + return 0, nil + } + + tr := emptyTransformable() + tr.header = resp.Header + + remaining := r.remaining.Execute(emptyTransformContext(), tr, "", r.log) + if remaining == "" { + return 0, errors.New("remaining value is empty") + } + m, err := strconv.ParseInt(remaining, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse rate-limit remaining value: %w", err) + } + + if m != 0 { + return 0, nil + } + + if r.reset == nil { + r.log.Warn("reset rate limit is not set") + return 0, nil + } + + reset := r.reset.Execute(emptyTransformContext(), tr, "", r.log) + if reset == "" { + return 0, errors.New("reset value is empty") + } + + epoch, err := strconv.ParseInt(reset, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) + } + + if timeNow().Sub(time.Unix(epoch, 0)) > 0 { + return 0, nil + } + + return epoch, nil +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go new file mode 100644 index 00000000000..db8b8760a25 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go @@ -0,0 +1,87 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "net/http" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// Test getRateLimit function with a remaining quota, expect to receive 0, nil. +func TestGetRateLimitCase1(t *testing.T) { + header := make(http.Header) + header.Add("X-Rate-Limit-Limit", "120") + header.Add("X-Rate-Limit-Remaining", "118") + header.Add("X-Rate-Limit-Reset", "1581658643") + tplLimit := &valueTpl{} + tplReset := &valueTpl{} + tplRemaining := &valueTpl{} + assert.NoError(t, tplLimit.Unpack(`{{.header.Get "X-Rate-Limit-Limit"}}`)) + assert.NoError(t, tplReset.Unpack(`{{.header.Get "X-Rate-Limit-Reset"}}`)) + assert.NoError(t, tplRemaining.Unpack(`{{.header.Get "X-Rate-Limit-Remaining"}}`)) + rateLimit := &rateLimiter{ + limit: tplLimit, + reset: tplReset, + remaining: tplRemaining, + } + resp := &http.Response{Header: header} + epoch, err := rateLimit.getRateLimit(resp) + assert.NoError(t, err) + assert.EqualValues(t, 0, epoch) +} + +// Test getRateLimit function with a past time, expect to receive 0, nil. +func TestGetRateLimitCase2(t *testing.T) { + header := make(http.Header) + header.Add("X-Rate-Limit-Limit", "10") + header.Add("X-Rate-Limit-Remaining", "0") + header.Add("X-Rate-Limit-Reset", "1581658643") + tplLimit := &valueTpl{} + tplReset := &valueTpl{} + tplRemaining := &valueTpl{} + assert.NoError(t, tplLimit.Unpack(`{{.header.Get "X-Rate-Limit-Limit"}}`)) + assert.NoError(t, tplReset.Unpack(`{{.header.Get "X-Rate-Limit-Reset"}}`)) + assert.NoError(t, tplRemaining.Unpack(`{{.header.Get "X-Rate-Limit-Remaining"}}`)) + rateLimit := &rateLimiter{ + limit: tplLimit, + reset: tplReset, + remaining: tplRemaining, + } + resp := &http.Response{Header: header} + epoch, err := rateLimit.getRateLimit(resp) + assert.NoError(t, err) + assert.EqualValues(t, 0, epoch) +} + +// Test getRateLimit function with a time yet to come, expect to receive , nil. +func TestGetRateLimitCase3(t *testing.T) { + epoch := int64(1604582732 + 100) + timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } + t.Cleanup(func() { timeNow = time.Now }) + + header := make(http.Header) + header.Add("X-Rate-Limit-Limit", "10") + header.Add("X-Rate-Limit-Remaining", "0") + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(epoch, 10)) + tplLimit := &valueTpl{} + tplReset := &valueTpl{} + tplRemaining := &valueTpl{} + assert.NoError(t, tplLimit.Unpack(`{{.header.Get "X-Rate-Limit-Limit"}}`)) + assert.NoError(t, tplReset.Unpack(`{{.header.Get "X-Rate-Limit-Reset"}}`)) + assert.NoError(t, tplRemaining.Unpack(`{{.header.Get "X-Rate-Limit-Remaining"}}`)) + rateLimit := &rateLimiter{ + limit: tplLimit, + reset: tplReset, + remaining: tplRemaining, + } + resp := &http.Response{Header: header} + epoch2, err := rateLimit.getRateLimit(resp) + assert.NoError(t, err) + assert.EqualValues(t, 1604582832, epoch2) +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index f19998cb664..c425f0aa95d 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -26,6 +26,26 @@ func registerRequestTransforms() { registerTransform(requestNamespace, setName, newSetRequest) } +type httpClient struct { + client *http.Client + limiter *rateLimiter +} + +func (c *httpClient) do(stdCtx context.Context, trCtx transformContext, req *http.Request) (*http.Response, error) { + resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { + return c.client.Do(req) + }) + if err != nil { + return nil, fmt.Errorf("failed to execute http client.Do: %w", err) + } + if resp.StatusCode > 399 { + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("server responded with status code %d: %s", resp.StatusCode, string(body)) + } + return resp, nil +} + func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []basicTransform) (*transformable, error) { req := emptyTransformable() req.url = url @@ -114,13 +134,13 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transform type requester struct { log *logp.Logger - client *http.Client + client *httpClient requestFactory *requestFactory responseProcessor *responseProcessor } func newRequester( - client *http.Client, + client *httpClient, requestFactory *requestFactory, responseProcessor *responseProcessor, log *logp.Logger) *requester { @@ -138,17 +158,12 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu return fmt.Errorf("failed to create http request: %w", err) } - httpResp, err := r.client.Do(req) + httpResp, err := r.client.do(stdCtx, trCtx, req) if err != nil { return fmt.Errorf("failed to execute http client.Do: %w", err) } defer httpResp.Body.Close() - if httpResp.StatusCode > 399 { - body, _ := ioutil.ReadAll(httpResp.Body) - return fmt.Errorf("server responded with status code %d: %s", httpResp.StatusCode, string(body)) - } - eventsCh, err := r.responseProcessor.startProcessing(stdCtx, trCtx, httpResp) if err != nil { return err From bedf52d9da6fddd59ab226216a932c74c67368e6 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 6 Nov 2020 16:56:45 +0100 Subject: [PATCH 22/35] Put back original filebeat.yml --- x-pack/filebeat/filebeat.yml | 299 +++++++++++++++++++++++++++++------ 1 file changed, 250 insertions(+), 49 deletions(-) diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 5ee3119091e..9a29bc76962 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -18,52 +18,253 @@ filebeat.inputs: # you can use different inputs for various configurations. # Below are the input specific configurations. -- type: httpjsonv2 - interval: 10s - request.method: post - request.url: https://patata.free.beeceptor.com - request.transforms: - - set: - target: body.foo - value: bazz - response.transforms: - - set: - target: body.foo - value: patata - response.pagination: - - set: - target: url.value - value: "https://patata.free.beeceptor.com/page" - - set: - target: url.params.p - value: "{{.last_response.body.page}}" - response.split: - target: body.foo - type: array|map - transforms: - - ... - split: - - -# limit split to be last (and to be just one) -# split can't have split transform itself -# iterate over maps or arrays dynamically -# keys_field to keep value of the key if a map is found -# add option to keep headers between redirects - -# { -# "bar": [ -# { -# "bazz": { -# "somekey": [ -# "bazinga": { -# "key": "value" -# } -# ] -# } -# } -# ] -# } - -output.console: - pretty: true +- type: log + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + + ### Multiline options + + # Multiline can be used for log messages spanning multiple lines. This is common + # for Java Stack Traces or C-Line Continuation + + # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [ + #multiline.pattern: ^\[ + + # Defines if the pattern set under pattern should be negated or not. Default is false. + #multiline.negate: false + + # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern + # that was (not) matched before or after or as long as a pattern is not matched based on negate. + # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash + #multiline.match: after + +# filestream is an experimental input. It is going to replace log input in the future. +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + +# ============================== Filebeat modules ============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +# ======================= Elasticsearch template setting ======================= + +setup.template.settings: + index.number_of_shards: 1 + #index.codec: best_compression + #_source.enabled: false + + +# ================================== General =================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging + +# ================================= Dashboards ================================= +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here or by using the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +# =================================== Kibana =================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + +# =============================== Elastic Cloud ================================ + +# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ================================== Outputs =================================== + +# Configure what output to use when sending the data collected by the beat. + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + +# ------------------------------ Logstash Output ------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +# ================================= Processors ================================= +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ + +# ================================== Logging =================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +# ============================= X-Pack Monitoring ============================== +# Filebeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + +# ============================== Instrumentation =============================== + +# Instrumentation support for the filebeat. +#instrumentation: + # Set to true to enable instrumentation of filebeat. + #enabled: false + + # Environment in which filebeat is running on (eg: staging, production, etc.) + #environment: "" + + # APM Server hosts to report instrumentation results to. + #hosts: + # - http://localhost:8200 + + # API Key for the APM Server(s). + # If api_key is set then secret_token will be ignored. + #api_key: + + # Secret token for the APM Server(s). + #secret_token: + + +# ================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: true + From 02dcf8ab687a354e8c2bd8c6d3b10f74a835d6f6 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 6 Nov 2020 17:04:16 +0100 Subject: [PATCH 23/35] Add deprecation warning for old httpjson version --- x-pack/filebeat/input/httpjson/input_manager.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/filebeat/input/httpjson/input_manager.go b/x-pack/filebeat/input/httpjson/input_manager.go index 31560839b0e..1efa41470b2 100644 --- a/x-pack/filebeat/input/httpjson/input_manager.go +++ b/x-pack/filebeat/input/httpjson/input_manager.go @@ -13,6 +13,7 @@ import ( cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" v2 "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson/internal/v2" ) @@ -45,6 +46,7 @@ func (m inputManager) Create(cfg *common.Config) (inputv2.Input, error) { if b, _ := cfg.Bool("is_v2", -1); b { return m.v2inputManager.Create(cfg) } + cfgwarn.Deprecate("7.12", "you are using a deprecated version of httpjson config") config := newDefaultConfig() if err := cfg.Unpack(&config); err != nil { return nil, err From afb68a7474eadb88e2ea4ce9b26fadc38ca99592 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 12 Nov 2020 15:27:48 +0100 Subject: [PATCH 24/35] Add transform tests --- .../input/httpjson/internal/v2/transform.go | 16 +- .../httpjson/internal/v2/transform_append.go | 4 +- .../internal/v2/transform_append_test.go | 186 ++++++++++++++++++ .../httpjson/internal/v2/transform_delete.go | 4 +- .../internal/v2/transform_delete_test.go | 182 +++++++++++++++++ .../httpjson/internal/v2/transform_set.go | 2 +- .../internal/v2/transform_set_test.go | 113 +++++++++++ .../httpjson/internal/v2/transform_target.go | 10 +- .../internal/v2/transform_target_test.go | 74 +++++++ .../httpjson/internal/v2/transform_test.go | 165 ++++++++++++++++ .../input/httpjson/internal/v2/value_tpl.go | 3 +- 11 files changed, 736 insertions(+), 23 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/transform_test.go diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform.go b/x-pack/filebeat/input/httpjson/internal/v2/transform.go index e0b8a86a114..18c0da78d70 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform.go @@ -8,7 +8,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "github.com/pkg/errors" @@ -65,8 +64,8 @@ func (t *transformable) clone() *transformable { func emptyTransformable() *transformable { return &transformable{ - body: make(common.MapStr), - header: make(http.Header), + body: common.MapStr{}, + header: http.Header{}, } } @@ -95,9 +94,8 @@ func newTransformsFromConfig(config transformsConfig, namespace string, log *log for _, tfConfig := range config { if len(tfConfig.GetFields()) != 1 { return nil, errors.Errorf( - "each transform must have exactly one action, but found %d actions (%v)", + "each transform must have exactly one action, but found %d actions", len(tfConfig.GetFields()), - strings.Join(tfConfig.GetFields(), ","), ) } @@ -141,11 +139,3 @@ func newBasicTransformsFromConfig(config transformsConfig, namespace string, log return rts, nil } - -func (trans transforms) String() string { - var s []string - for _, p := range trans { - s = append(s, p.transformName()) - } - return strings.Join(s, ", ") -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go index ebe9e821b74..f264f6a22af 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go @@ -145,6 +145,8 @@ func appendHeader(ctx transformContext, transformable *transformable, key, value } func appendURLParams(ctx transformContext, transformable *transformable, key, value string) error { - transformable.url.Query().Add(key, value) + q := transformable.url.Query() + q.Add(key, value) + transformable.url.RawQuery = q.Encode() return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go new file mode 100644 index 00000000000..cef04fa034f --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go @@ -0,0 +1,186 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestNewAppend(t *testing.T) { + cases := []struct { + name string + constructor constructor + config map[string]interface{} + expectedTarget targetInfo + expectedErr string + }{ + { + name: "newAppendResponse targets body", + constructor: newAppendResponse, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newAppendResponse targets something else", + constructor: newAppendResponse, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + { + name: "newAppendRequest targets body", + constructor: newAppendRequest, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newAppendRequest targets header", + constructor: newAppendRequest, + config: map[string]interface{}{ + "target": "header.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "header"}, + }, + { + name: "newAppendRequest targets url param", + constructor: newAppendRequest, + config: map[string]interface{}{ + "target": "url.params.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "url.params"}, + }, + { + name: "newAppendRequest targets something else", + constructor: newAppendRequest, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + { + name: "newAppendPagination targets body", + constructor: newAppendPagination, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newAppendPagination targets header", + constructor: newAppendPagination, + config: map[string]interface{}{ + "target": "header.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "header"}, + }, + { + name: "newAppendPagination targets url param", + constructor: newAppendPagination, + config: map[string]interface{}{ + "target": "url.params.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "url.params"}, + }, + { + name: "newAppendPagination targets url value", + constructor: newAppendPagination, + config: map[string]interface{}{ + "target": "url.value", + }, + expectedErr: "invalid target type: url.value", + }, + { + name: "newAppendPagination targets something else", + constructor: newAppendPagination, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + cfg := common.MustNewConfigFrom(tc.config) + gotAppend, gotErr := tc.constructor(cfg, nil) + if tc.expectedErr == "" { + assert.NoError(t, gotErr) + assert.Equal(t, tc.expectedTarget, (gotAppend.(*appendt)).targetInfo) + } else { + assert.EqualError(t, gotErr, tc.expectedErr) + } + }) + } +} + +func TestAppendFunctions(t *testing.T) { + cases := []struct { + name string + tfunc func(ctx transformContext, transformable *transformable, key, val string) error + paramCtx transformContext + paramTr *transformable + paramKey string + paramVal string + expectedTr *transformable + expectedErr error + }{ + { + name: "appendBody", + tfunc: appendBody, + paramCtx: transformContext{}, + paramTr: &transformable{body: common.MapStr{"a_key": "a_value"}}, + paramKey: "a_key", + paramVal: "another_value", + expectedTr: &transformable{body: common.MapStr{"a_key": []interface{}{"a_value", "another_value"}}}, + expectedErr: nil, + }, + { + name: "appendHeader", + tfunc: appendHeader, + paramCtx: transformContext{}, + paramTr: &transformable{header: http.Header{ + "A_key": []string{"a_value"}, + }}, + paramKey: "a_key", + paramVal: "another_value", + expectedTr: &transformable{header: http.Header{"A_key": []string{"a_value", "another_value"}}}, + expectedErr: nil, + }, + { + name: "appendURLParams", + tfunc: appendURLParams, + paramCtx: transformContext{}, + paramTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value")}, + paramKey: "a_key", + paramVal: "another_value", + expectedTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value&a_key=another_value")}, + expectedErr: nil, + }, + } + + for _, tcase := range cases { + tcase := tcase + t.Run(tcase.name, func(t *testing.T) { + gotErr := tcase.tfunc(tcase.paramCtx, tcase.paramTr, tcase.paramKey, tcase.paramVal) + if tcase.expectedErr == nil { + assert.NoError(t, gotErr) + } else { + assert.EqualError(t, gotErr, tcase.expectedErr.Error()) + } + assert.EqualValues(t, tcase.expectedTr, tcase.paramTr) + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go index a00715ae4c9..00d29d3fa72 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go @@ -123,6 +123,8 @@ func deleteHeader(ctx transformContext, transformable *transformable, key string } func deleteURLParams(ctx transformContext, transformable *transformable, key string) error { - transformable.url.Query().Del(key) + q := transformable.url.Query() + q.Del(key) + transformable.url.RawQuery = q.Encode() return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go new file mode 100644 index 00000000000..4d493d241e4 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go @@ -0,0 +1,182 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestNewDelete(t *testing.T) { + cases := []struct { + name string + constructor constructor + config map[string]interface{} + expectedTarget targetInfo + expectedErr string + }{ + { + name: "newDeleteResponse targets body", + constructor: newDeleteResponse, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newDeleteResponse targets something else", + constructor: newDeleteResponse, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + { + name: "newDeleteRequest targets body", + constructor: newDeleteRequest, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newDeleteRequest targets header", + constructor: newDeleteRequest, + config: map[string]interface{}{ + "target": "header.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "header"}, + }, + { + name: "newDeleteRequest targets url param", + constructor: newDeleteRequest, + config: map[string]interface{}{ + "target": "url.params.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "url.params"}, + }, + { + name: "newDeleteRequest targets something else", + constructor: newDeleteRequest, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + { + name: "newDeletePagination targets body", + constructor: newDeletePagination, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newDeletePagination targets header", + constructor: newDeletePagination, + config: map[string]interface{}{ + "target": "header.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "header"}, + }, + { + name: "newDeletePagination targets url param", + constructor: newDeletePagination, + config: map[string]interface{}{ + "target": "url.params.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "url.params"}, + }, + { + name: "newDeletePagination targets url value", + constructor: newDeletePagination, + config: map[string]interface{}{ + "target": "url.value", + }, + expectedErr: "invalid target type: url.value", + }, + { + name: "newDeletePagination targets something else", + constructor: newDeletePagination, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + cfg := common.MustNewConfigFrom(tc.config) + gotDelete, gotErr := tc.constructor(cfg, nil) + if tc.expectedErr == "" { + assert.NoError(t, gotErr) + assert.Equal(t, tc.expectedTarget, (gotDelete.(*delete)).targetInfo) + } else { + assert.EqualError(t, gotErr, tc.expectedErr) + } + }) + } +} + +func TestDeleteFunctions(t *testing.T) { + cases := []struct { + name string + tfunc func(ctx transformContext, transformable *transformable, key string) error + paramCtx transformContext + paramTr *transformable + paramKey string + expectedTr *transformable + expectedErr error + }{ + { + name: "deleteBody", + tfunc: deleteBody, + paramCtx: transformContext{}, + paramTr: &transformable{body: common.MapStr{"a_key": "a_value"}}, + paramKey: "a_key", + expectedTr: &transformable{body: common.MapStr{}}, + expectedErr: nil, + }, + { + name: "deleteHeader", + tfunc: deleteHeader, + paramCtx: transformContext{}, + paramTr: &transformable{header: http.Header{ + "A_key": []string{"a_value"}, + }}, + paramKey: "a_key", + expectedTr: &transformable{header: http.Header{}}, + expectedErr: nil, + }, + { + name: "deleteURLParams", + tfunc: deleteURLParams, + paramCtx: transformContext{}, + paramTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value")}, + paramKey: "a_key", + expectedTr: &transformable{url: newURL("http://foo.example.com")}, + expectedErr: nil, + }, + } + + for _, tcase := range cases { + tcase := tcase + t.Run(tcase.name, func(t *testing.T) { + gotErr := tcase.tfunc(tcase.paramCtx, tcase.paramTr, tcase.paramKey) + if tcase.expectedErr == nil { + assert.NoError(t, gotErr) + } else { + assert.EqualError(t, gotErr, tcase.expectedErr.Error()) + } + assert.EqualValues(t, tcase.expectedTr, tcase.paramTr) + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go index 4da04799db5..691885fc5b0 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go @@ -138,7 +138,7 @@ func setHeader(ctx transformContext, transformable *transformable, key, value st func setURLParams(ctx transformContext, transformable *transformable, key, value string) error { q := transformable.url.Query() - q.Add(key, value) + q.Set(key, value) transformable.url.RawQuery = q.Encode() return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go index 40d207c3a20..31493353b7d 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go @@ -14,6 +14,119 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +func TestNewSet(t *testing.T) { + cases := []struct { + name string + constructor constructor + config map[string]interface{} + expectedTarget targetInfo + expectedErr string + }{ + { + name: "newSetResponse targets body", + constructor: newSetResponse, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newSetResponse targets something else", + constructor: newSetResponse, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + { + name: "newSetRequest targets body", + constructor: newSetRequest, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newSetRequest targets header", + constructor: newSetRequest, + config: map[string]interface{}{ + "target": "header.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "header"}, + }, + { + name: "newSetRequest targets url param", + constructor: newSetRequest, + config: map[string]interface{}{ + "target": "url.params.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "url.params"}, + }, + { + name: "newSetRequest targets something else", + constructor: newSetRequest, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + { + name: "newSetPagination targets body", + constructor: newSetPagination, + config: map[string]interface{}{ + "target": "body.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "body"}, + }, + { + name: "newSetPagination targets header", + constructor: newSetPagination, + config: map[string]interface{}{ + "target": "header.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "header"}, + }, + { + name: "newSetPagination targets url param", + constructor: newSetPagination, + config: map[string]interface{}{ + "target": "url.params.foo", + }, + expectedTarget: targetInfo{Name: "foo", Type: "url.params"}, + }, + { + name: "newSetPagination targets url value", + constructor: newSetPagination, + config: map[string]interface{}{ + "target": "url.value", + }, + expectedTarget: targetInfo{Type: "url.value"}, + }, + { + name: "newSetPagination targets something else", + constructor: newSetPagination, + config: map[string]interface{}{ + "target": "cursor.foo", + }, + expectedErr: "invalid target: cursor.foo", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + cfg := common.MustNewConfigFrom(tc.config) + gotSet, gotErr := tc.constructor(cfg, nil) + if tc.expectedErr == "" { + assert.NoError(t, gotErr) + assert.Equal(t, tc.expectedTarget, (gotSet.(*set)).targetInfo) + } else { + assert.EqualError(t, gotErr, tc.expectedErr) + } + }) + } +} + func TestSetFunctions(t *testing.T) { cases := []struct { name string diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go index 630976aae9a..2fd6d83d3c0 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_target.go @@ -13,7 +13,6 @@ type targetType string const ( targetBody targetType = "body" - targetCursor targetType = "cursor" targetHeader targetType = "header" targetURLValue targetType = "url.value" targetURLParams targetType = "url.params" @@ -44,6 +43,10 @@ func getTargetInfo(t string) (targetInfo, error) { } paramParts := strings.SplitN(parts[1], ".", 2) + if len(paramParts) < 2 || paramParts[0] != "params" { + return targetInfo{}, errInvalidTarget{t} + } + return targetInfo{ Type: targetURLParams, Name: paramParts[1], @@ -58,11 +61,6 @@ func getTargetInfo(t string) (targetInfo, error) { Type: targetBody, Name: parts[1], }, nil - case "cursor": - return targetInfo{ - Type: targetCursor, - Name: parts[1], - }, nil } return targetInfo{}, errInvalidTarget{t} } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go new file mode 100644 index 00000000000..2042c8dab38 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_target_test.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetTargetInfo(t *testing.T) { + cases := []struct { + name string + param string + expected targetInfo + expectedErr string + }{ + { + name: "valid url.value", + param: "url.value", + expected: targetInfo{Type: "url.value"}, + }, + { + name: "invalid url.value", + param: "url.value.something", + expectedErr: "invalid target: url.value.something", + }, + { + name: "valid url.params", + param: "url.params.foo", + expected: targetInfo{Type: "url.params", Name: "foo"}, + }, + { + name: "invalid url.params", + param: "url.params", + expectedErr: "invalid target: url.params", + }, + { + name: "valid header", + param: "header.foo", + expected: targetInfo{Type: "header", Name: "foo"}, + }, + { + name: "valid body", + param: "body.foo.bar", + expected: targetInfo{Type: "body", Name: "foo.bar"}, + }, + { + name: "invalid target: missing part", + param: "header", + expectedErr: "invalid target: header", + }, + { + name: "invalid target: unknown", + param: "unknown.foo", + expectedErr: "invalid target: unknown.foo", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + got, gotErr := getTargetInfo(tc.param) + if tc.expectedErr == "" { + assert.NoError(t, gotErr) + assert.Equal(t, tc.expected, got) + } else { + assert.EqualError(t, gotErr, tc.expectedErr) + } + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go new file mode 100644 index 00000000000..a102c3b9bd8 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go @@ -0,0 +1,165 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "net/http" + "testing" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/stretchr/testify/assert" +) + +func TestEmptyTransformContext(t *testing.T) { + ctx := emptyTransformContext() + assert.Equal(t, &cursor{}, ctx.cursor) + assert.Equal(t, &common.MapStr{}, ctx.lastEvent) + assert.Equal(t, emptyTransformable(), ctx.lastResponse) +} + +func TestEmptyTransformable(t *testing.T) { + tr := emptyTransformable() + assert.Equal(t, common.MapStr{}, tr.body) + assert.Equal(t, http.Header{}, tr.header) +} + +func TestTransformableNilClone(t *testing.T) { + var tr *transformable + cl := tr.clone() + assert.Equal(t, common.MapStr{}, cl.body) + assert.Equal(t, http.Header{}, cl.header) +} + +func TestTransformableClone(t *testing.T) { + tr := emptyTransformable() + _, _ = tr.body.Put("key", "value") + cl := tr.clone() + assert.Equal(t, common.MapStr{"key": "value"}, cl.body) + assert.Equal(t, http.Header{}, cl.header) +} + +func TestNewTransformsFromConfig(t *testing.T) { + registerTransform("test", setName, newSetRequest) + t.Cleanup(func() { registeredTransforms = newRegistry() }) + + cases := []struct { + name string + paramCfg map[string]interface{} + paramNamespace string + expectedTransforms transforms + expectedErr string + }{ + { + name: "fails if config has more than one action", + paramCfg: map[string]interface{}{ + "set": nil, + "set2": nil, + }, + expectedErr: "each transform must have exactly one action, but found 2 actions", + }, + { + name: "fails if not found in namespace", + paramCfg: map[string]interface{}{ + "set": nil, + }, + paramNamespace: "empty", + expectedErr: "the transform set does not exist. Valid transforms: test: (set)\n", + }, + { + name: "fails if constructor fails", + paramCfg: map[string]interface{}{ + "set": map[string]interface{}{ + "target": "invalid", + }, + }, + paramNamespace: "test", + expectedErr: "invalid target: invalid", + }, + { + name: "transform is correct", + paramCfg: map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.foo", + }, + }, + paramNamespace: "test", + expectedTransforms: transforms{ + &set{ + targetInfo: targetInfo{Name: "foo", Type: "body"}, + }, + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + cfg := common.MustNewConfigFrom(tc.paramCfg) + gotTransforms, gotErr := newTransformsFromConfig(transformsConfig{cfg}, tc.paramNamespace, nil) + if tc.expectedErr == "" { + assert.NoError(t, gotErr) + tr := gotTransforms[0].(*set) + tr.runFunc = nil // we do not want to check func pointer + assert.EqualValues(t, tc.expectedTransforms, gotTransforms) + } else { + assert.EqualError(t, gotErr, tc.expectedErr) + } + }) + } +} + +type fakeTransform struct{} + +func (fakeTransform) transformName() string { return "fake" } + +func TestNewBasicTransformsFromConfig(t *testing.T) { + fakeConstr := func(*common.Config, *logp.Logger) (transform, error) { + + return fakeTransform{}, nil + } + + registerTransform("test", setName, newSetRequest) + registerTransform("test", "fake", fakeConstr) + t.Cleanup(func() { registeredTransforms = newRegistry() }) + + cases := []struct { + name string + paramCfg map[string]interface{} + paramNamespace string + expectedErr string + }{ + { + name: "succeeds if transform is basicTransform", + paramCfg: map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.foo", + }, + }, + paramNamespace: "test", + }, + { + name: "fails if transform is not a basicTransform", + paramCfg: map[string]interface{}{ + "fake": nil, + }, + paramNamespace: "test", + expectedErr: "transform fake is not a valid test transform", + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + cfg := common.MustNewConfigFrom(tc.paramCfg) + _, gotErr := newBasicTransformsFromConfig(transformsConfig{cfg}, tc.paramNamespace, nil) + if tc.expectedErr == "" { + assert.NoError(t, gotErr) + } else { + assert.EqualError(t, gotErr, tc.expectedErr) + } + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index fe891467553..c6bac668141 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -7,6 +7,7 @@ package v2 import ( "bytes" "regexp" + "strings" "text/template" "time" @@ -70,7 +71,7 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal } val = buf.String() - if val == "" { + if val == "" || strings.Contains(val, "") { val = defaultVal } return val From 6789ae3cfe72db57dbbe6c7805a4e3eb4999fdca Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 23 Nov 2020 13:13:14 +0100 Subject: [PATCH 25/35] Initial doc changes --- .../docs/inputs/input-httpjson.asciidoc | 959 +++++++++++++----- x-pack/filebeat/filebeat.yml | 2 +- .../input/httpjson/internal/v2/config.go | 17 +- .../httpjson/internal/v2/config_request.go | 23 +- .../input/httpjson/internal/v2/input.go | 10 +- .../httpjson/internal/v2/rate_limiter.go | 4 +- .../input/httpjson/internal/v2/split.go | 15 +- .../input/httpjson/internal/v2/split_test.go | 56 + .../httpjson/internal/v2/transform_append.go | 4 +- .../httpjson/internal/v2/transform_set.go | 4 +- .../httpjson/internal/v2/transform_test.go | 3 +- .../input/httpjson/internal/v2/value_tpl.go | 27 +- .../httpjson/internal/v2/value_tpl_test.go | 21 +- 13 files changed, 828 insertions(+), 317 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 24e673d09e6..e914b990bee 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -13,20 +13,18 @@ beta[] Use the `httpjson` input to read messages from an HTTP API with JSON payloads. -For example, this input is used to retrieve MISP threat indicators in the -Filebeat <> module. - -This input supports retrieval at a configurable interval and pagination. +This input supports basic auth, oauth2, retrieval at a configurable interval, pagination, retries and rate limiting. Example configurations: ["source","yaml",subs="attributes"] ---- -{beatname_lc}.inputs: +filebeat.inputs: # Fetch your public IP every minute. - type: httpjson - url: https://api.ipify.org/?format=json + is_v2: true interval: 1m + request.url: https://api.ipify.org/?format=json processors: - decode_json_fields fields: [message] @@ -35,361 +33,229 @@ Example configurations: ["source","yaml",subs="attributes"] ---- -{beatname_lc}.inputs: +filebeat.inputs: - type: httpjson - url: http://localhost:9200/_search?scroll=5m - http_method: POST - json_objects_array: hits.hits - pagination: - extra_body_content: - scroll: 5m - id_field: _scroll_id - req_field: scroll_id - url: http://localhost:9200/_search/scroll + is_v2: true + request.url: http://localhost:9200/_search?scroll=5m + request.method: POST + response.split: + target: body.hits.hits + response.pagination: + - set: + target: url.value + value: http://localhost:9200/_search/scroll + - set: + target: .url.params.scroll_id + value: '{{.last_request.body._scroll_id}}' + - set: + target: .body.scroll + value: 5m ---- -Additionally, it supports authentication via HTTP Headers, API key or oauth2. +Additionally, it supports authentication via Basic auth, HTTP Headers or oauth2. Example configurations with authentication: ["source","yaml",subs="attributes"] ---- -{beatname_lc}.inputs: +filebeat.inputs: - type: httpjson - http_headers: - Authorization: 'Basic aGVsbG86d29ybGQ=' - url: http://localhost + is_v2: true + request.url: http://localhost + request.transforms: + - set: + target: header.Authorization + value: 'Basic aGVsbG86d29ybGQ=' ---- ["source","yaml",subs="attributes"] ---- -{beatname_lc}.inputs: +filebeat.inputs: - type: httpjson - oauth2: + is_v2: true + auth.oauth2: client.id: 12345678901234567890abcdef client.secret: abcdef12345678901234567890 token_url: http://localhost/oauth2/token - url: http://localhost + request.url: http://localhost ---- -==== Configuration options +==== Input state -The `httpjson` input supports the following configuration options plus the -<<{beatname_lc}-input-{type}-common-options>> described later. +The `httpjson` input will keep a runtime state between requests, that can be accessed by some configuration options and transfroms. -[float] -==== `api_key` +The state has the following elements: -API key to access the HTTP API. When set, this adds an `Authorization` header to -the HTTP request with this as the value. +- `last_response.url.value`: The full URL with params and fragments from the last request with a successful response. +- `last_request.url.params`: A map containing the params from the URL in `last_response.url.value`. +- `last_response.header`: A map containing the headers from the last successful response. +- `last_response.body`: A map containing the parsed JSON body from the last successful response. (before any transform) +- `last_event`: A map representing the last event sent to the output (result from applying transforms to `last_response.body`). +- `url.value`: The full URL with params and fragments. Will make reference to the request URL for configs under `request.*` and for the last successful request under `response.*`. +- `url.params`: A map containing the URL params. Will make reference to the request URL for configs under `request.*` and for the last successful request under `response.*`. +- `header`: A map containing the headers. Will make reference to the request headers for configs under `request.*` and for the last successful response ones under `response.*`. +- `body`: A map containing the body. Will make reference to the request body for configs under `request.*` and for the last successful response one under `response.*`. +- `cursor`: A map containing any data the user configured to be stored between restarts. -[float] -==== `http_client_timeout` - -Duration before declaring that the HTTP client connection has timed out. -Defaults to `60s`. Valid time units are `ns`, `us`, `ms`, `s` (default), `m`, -`h`. +All of the mentioned objects are only stored at runtime, except `cursor` which values are persisted between restarts. -[float] -==== `http_headers` +==== Transforms -Additional HTTP headers to set in the requests. The default value is `null` -(no additional headers). - -["source","yaml",subs="attributes"] ----- -- type: httpjson - http_headers: - Authorization: 'Basic aGVsbG86d29ybGQ=' ----- - -[float] -==== `http_method` - -HTTP method to use when making requests. `GET` or `POST` are the options. -Defaults to `GET`. +A transform is an action that let the user modify the input state. Depending on where the transform is defined, it will have access for reading or writing different elements of the state. +This access limitations will be specified in the corresponding configuration sections. [float] -==== `http_request_body` +==== `append` -An optional HTTP POST body. The configuration value must be an object, and it -will be encoded to JSON. This is only valid when `http_method` is `POST`. -Defaults to `null` (no HTTP body). +Append will append a value to a list. If the field does not exist the first entry will be a scalar value, and subsequent additions will convert it to a list. ["source","yaml",subs="attributes"] ---- -- type: httpjson - http_method: POST - http_request_body: - query: - bool: - filter: - term: - type: authentication +- append: + target: body.foo.bar + value: '{{.cursor.baz}}' + default: "a default value" ---- -[float] -==== `interval` - -Duration between repeated requests. By default, the interval is `0` which means -it performs a single request then stops. It may make additional pagination -requests in response to the initial request if pagination is enabled. +- `target` defines the destination field where the value is stored. +- `value` defines the value that will be stored and it is a value template. +- `default` defines the fallback value whenever `value` is empty or the template parsing fails. Default templates do not have access to any state, only to functions. [float] -==== `json_objects_array` +==== `delete` -If the response body contains a JSON object containing an array then this option -specifies the key containing that array. Each object in that array will generate -an event. This example response contains an array called `events` that we want -to index. - -["source","json",subs="attributes"] ----- -{ - "time": "2020-06-02 23:22:32 UTC", - "events": [ - { - "timestamp": "2020-05-02 11:10:03 UTC", - "event": { - "category": "authorization" - }, - "user": { - "name": "fflintstone" - } - }, - { - "timestamp": "2020-05-05 13:03:11 UTC", - "event": { - "category": "authorization" - }, - "user": { - "name": "brubble" - } - } - ] -} ----- - -The config needs to specify `events` as the `json_objects_array` value. +Delete will delete the target field. ["source","yaml",subs="attributes"] ---- -- type: httpjson - json_objects_array: events +- delete: + target: body.foo.bar ---- -[float] -==== `split_events_by` +- `target` defines the destination field to delete. -If the response body contains a JSON object containing an array then this option -specifies the key containing that array. Each object in that array will generate -an event, but will maintain the common fields of the document as well. +NOTE: If `target` is a list, it will delete the list completely, and not a single element. -["source","json",subs="attributes"] ----- -{ - "time": "2020-06-02 23:22:32 UTC", - "user": "Bob", - "events": [ - { - "timestamp": "2020-05-02 11:10:03 UTC", - "event": { - "category": "authorization" - } - }, - { - "timestamp": "2020-05-05 13:03:11 UTC", - "event": { - "category": "authorization" - } - } - ] -} ----- +[float] +==== `set` -The config needs to specify `events` as the `split_events_by` value. +Set will set a value. ["source","yaml",subs="attributes"] ---- -- type: httpjson - split_events_by: events ----- - -And will output the following events: - -["source","json",subs="attributes"] ----- -[ - { - "time": "2020-06-02 23:22:32 UTC", - "user": "Bob", - "events": { - "timestamp": "2020-05-02 11:10:03 UTC", - "event": { - "category": "authorization" - } - } - }, - { - "time": "2020-06-02 23:22:32 UTC", - "user": "Bob", - "events": { - "timestamp": "2020-05-05 13:03:11 UTC", - "event": { - "category": "authorization" - } - } - } -] +- set: + target: body.foo.bar + value: '{{.cursor.baz}}' + default: "a default value" ---- -It can be used in combination with `json_objects_array`, which will look for the field inside each element. +- `target` defines the destination field where the value is stored. +- `value` defines the value that will be stored and it is a value template. +- `default` defines the fallback value whenever `value` is empty or the template parsing fails. Default templates do not have access to any state, only to functions. -[float] -==== `no_http_body` +==== Value templates -Force HTTP requests to be sent with an empty HTTP body. Defaults to `false`. -This option cannot be used with `http_request_body`, -`pagination.extra_body_content`, or `pagination.req_field`. +Some configuration options and transforms can use value templates. Value templates are Go templates with access to the input state and to some built in functions. -[float] -==== `pagination.enabled` +The state elements and what operations can be performed are defined by the option or transform using them and will be specified in them. -The `enabled` setting can be used to disable the pagination configuration by -setting it to `false`. The default value is `true`. - -NOTE: Pagination settings are disabled if either `enabled` is set to `false` or -the `pagination` section is missing. - -[float] -==== `pagination.extra_body_content` - -An object containing additional fields that should be included in the pagination -request body. Defaults to `null`. +A value template looks like: ["source","yaml",subs="attributes"] ---- -- type: httpjson - pagination.extra_body_content: - max_items: 500 +- set: + target: body.foo.bar + value: '{{.cursor.baz}} more data' + default: "a default value" ---- -[float] -==== `pagination.header.field_name` - -The name of the HTTP header in the response that is used for pagination control. -The header value will be extracted from the response and used to make the next -pagination response. `pagination.header.regex_pattern` can be used to select -a subset of the value. - -[float] -==== `pagination.header.regex_pattern` - -The regular expression pattern to use for retrieving the pagination information -from the HTTP header field specified above. The first match becomes as the -value. - -[float] -==== `pagination.id_field` - -The name of a field in the JSON response body to use as the pagination ID. -The value will be included in the next pagination request under the key -specified by the `pagination.req_field` value. - -[float] -==== `pagination.req_field` +What is between `{{` `}}` will be evaluated. For more information on Go templates please refer to https://golang.org/pkg/text/template[the Go docs]. -The name of the field to include in the pagination JSON request body containing -the pagination ID defined by the `pagination.id_field` field. +Some built in helper functions are provided to work with the input state inside value templates: -[float] -==== `pagination.url` +- `parseDuration`: parses duration strings and return `time.Duration`. Example: `{{parseDuration "1h"}}`. +- `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `{{now (parseDuration "-1h")}}` returns the time at 1 hour before now. +- `parseTimestamp`: parses a timestamp in seconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732}}` returns `2020-11-05 13:25:32 +0000 UTC`. +- `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732000}}` returns `2020-11-05 13:25:32 +0000 UTC`. +- `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732000000000}}` returns `2020-11-05 13:25:32 +0000 UTC`. +- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Go lang predefined layouts or a custom one. Example: `{{ parseDate "2020-11-05T12:25:32Z" }}`, `{{ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" }}`, `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC }}`. +- `formatDate`: formats a `time.Time`. By default the format layout is `RFC3339` but optionally can accept any of the Go lang predefined layouts or a custom one. It will default to UTC timezone when formatting but optionally a different timezone can be specified. If the timezone is incorrect will default to UTC. Example: `{{ formatDate (now) "UnixDate" }}`, `{{ formatDate (now) "UnixDate" "America/New_York" }}`. +- `getRFC5988Link`: it extracts a specific relation from a list of https://tools.ietf.org/html/rfc5988[RFC5988] links. It is useful when we are parsing header values for pagination, for example. Example: `{{ getRFC5988Link "next" .last_response.header.Link }}` -This specifies the URL for sending pagination requests. Defaults to the `url` -value. This is only needed when the pagination requests need to be routed to -a different URL. +In addition to the provided functions, any of the native functions for `time.Time` and `http.Header` types can be used on the corresponding objects. Examples: `{{(now).Day}}`, `{{.last_response.header.Get "key"}}` -[float] -==== `rate_limit.limit` +==== Configuration options -This specifies the field in the HTTP header of the response that specifies the -total limit. +The `httpjson` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. [float] -==== `rate_limit.remaining` +==== `is_v2` -This specifies the field in the HTTP header of the response that specifies the -remaining quota of the rate limit. +Defines if the configuration is in the new `v2` format or not. Default: `false`. -[float] -==== `rate_limit.reset` - -This specifies the field in the HTTP Header of the response that specifies the -epoch time when the rate limit will reset. +NOTE: This defaulting to `false` is just to avoid breaking current configurations. V1 configuration is deprecated and will be unsupported in next releases. Any new configuration should use `is_v2: true`. [float] -==== `retry.max_attempts` +==== `interval` -This specifies the maximum number of retries for the retryable HTTP client. Default: 5. +Duration between repeated requests. It may make additional pagination requests in response to the initial request if pagination is enabled. Default: `60s`. [float] -==== `retry.wait_min` +==== `auth.basic.enabled` -This specifies the minimum time to wait before a retry is attempted. Default: 1s. +The `enabled` setting can be used to disable the basic auth configuration by +setting it to `false`. Default: `true`. -[float] -==== `retry.wait_max` - -This specifies the maximum time to wait before a retry is attempted. Default: 60s. +NOTE: Basic auth settings are disabled if either `enabled` is set to `false` or +the `auth.basic` section is missing. [float] -==== `ssl` +==== `auth.basic.user` -This specifies SSL/TLS configuration. If the ssl section is missing, the host's -CAs are used for HTTPS connections. See <> for more -information. +The `user` setting sets the user to authenticate with. [float] -==== `url` +==== `auth.basic.password` -The URL of the HTTP API. Required. +The `password` setting sets the password to use. [float] -==== `oauth2.enabled` +==== `auth.oauth2.enabled` The `enabled` setting can be used to disable the oauth2 configuration by setting it to `false`. The default value is `true`. NOTE: OAuth2 settings are disabled if either `enabled` is set to `false` or -the `oauth2` section is missing. +the `auth.oauth2` section is missing. [float] -==== `oauth2.provider` +==== `auth.oauth2.provider` The `provider` setting can be used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. Supported providers are: `azure`, `google`. [float] -==== `oauth2.client.id` +==== `auth.oauth2.client.id` The `client.id` setting is used as part of the authentication flow. It is always required except if using `google` as provider. Required for providers: `default`, `azure`. [float] -==== `oauth2.client.secret` +==== `auth.oauth2.client.secret` The `client.secret` setting is used as part of the authentication flow. It is always required except if using `google` as provider. Required for providers: `default`, `azure`. [float] -==== `oauth2.scopes` +==== `auth.oauth2.scopes` The `scopes` setting defines a list of scopes that will be requested during the oauth2 flow. It is optional for all providers. [float] -==== `oauth2.token_url` +==== `auth.oauth2.token_url` The `token_url` setting specifies the endpoint that will be used to generate the tokens during the oauth2 flow. It is required if no provider is specified. @@ -397,7 +263,7 @@ tokens during the oauth2 flow. It is required if no provider is specified. NOTE: For `azure` provider either `token_url` or `azure.tenant_id` is required. [float] -==== `oauth2.endpoint_params` +==== `auth.oauth2.endpoint_params` The `endpoint_params` setting specifies a set of values that will be sent on each request to the `token_url`. Each param key can have multiple values. @@ -406,7 +272,8 @@ Can be set for all providers except `google`. ["source","yaml",subs="attributes"] ---- - type: httpjson - oauth2: + is_v2: true + auth.oauth2: endpoint_params: Param1: - ValueA @@ -416,7 +283,7 @@ Can be set for all providers except `google`. ---- [float] -==== `oauth2.azure.tenant_id` +==== `auth.oauth2.azure.tenant_id` The `azure.tenant_id` is used for authentication when using `azure` provider. Since it is used in the process to generate the `token_url`, it can't be used in @@ -426,13 +293,13 @@ For information about where to find it, you can refer to https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal. [float] -==== `oauth2.azure.resource` +==== `auth.oauth2.azure.resource` The `azure.resource` is used to identify the accessed WebAPI resource when using `azure` provider. It is not required. [float] -==== `oauth2.google.credentials_file` +==== `auth.oauth2.google.credentials_file` The `google.credentials_file` setting specifies the credentials file for Google. @@ -441,7 +308,7 @@ default credentials from the environment will be attempted via ADC. For more inf how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [float] -==== `oauth2.google.credentials_json` +==== `auth.oauth2.google.credentials_json` The `google.credentials_json` setting allows to write your credentials information as raw JSON. @@ -450,7 +317,7 @@ default credentials from the environment will be attempted via ADC. For more inf how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. [float] -==== `oauth2.google.jwt_file` +==== `auth.oauth2.google.jwt_file` The `google.jwt_file` setting specifies the JWT Account Key file for Google. @@ -458,6 +325,582 @@ NOTE: Only one of the credentials settings can be set at once. If none is provid default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. +[float] +==== `request.url` + +The URL of the HTTP API. Required. + +[float] +==== `request.method` + +HTTP method to use when making requests. `GET` or `POST` are the options. Default: `GET`. + +[float] +==== `request.body` + +An optional HTTP POST body. The configuration value must be an object, and it +will be encoded to JSON. This is only valid when `request.method` is `POST`. +Defaults to `null` (no HTTP body). + +["source","yaml",subs="attributes"] +---- +- type: httpjson + is_v2: true + request.method: POST + request.body: + query: + bool: + filter: + term: + type: authentication +---- + +[float] +==== `request.timeout` + +Duration before declaring that the HTTP client connection has timed out. Valid time units are `ns`, `us`, `ms`, `s`, `m`, `h`. Default: `30s`. + +[float] +==== `request.ssl` + +This specifies SSL/TLS configuration. If the ssl section is missing, the host's +CAs are used for HTTPS connections. See <> for more +information. + +[float] +==== `request.retry.max_attempts` + +This specifies the maximum number of retries for the HTTP client. Default: `5`. + +[float] +==== `request.retry.wait_min` + +This specifies the minimum time to wait before a retry is attempted. Default: `1s`. + +[float] +==== `request.retry.wait_max` + +This specifies the maximum time to wait before a retry is attempted. Default: `60s`. + +[float] +==== `request.redirect.forward_headers` + +This specifies if headers are forwarded in case of a redirect. Default: `true`. + +[float] +==== `request.redirect.headers_ban_list` + +When `redirect.forward_headers` is set to `true`, all headers __except__ the ones defined in this list, will be forwarded. Default: `["WWW-Authenticate", "Authorization"]`. + +[float] +==== `request.redirect.max_redirects` + +Sets the maximum number of redirects to follow for a request. Default: `10`. + +[float] +==== `request.rate_limit.limit` + +This specifies the value of the response that specifies the total limit. It is defined with a Go template value. Can read state from: [`.last_response.header`] + +[float] +==== `request.rate_limit.remaining` + +This specifies the value of the response that specifies the remaining quota of the rate limit. It is defined with a Go template value. Can read state from: [`.last_response.header`] + +[float] +==== `request.rate_limit.reset` + +This specifies the value of the response that specifies the epoch time when the rate limit will reset. It is defined with a Go template value. Can read state from: [`.last_response.header`] + +[float] +==== `request.transforms` + +List of transforms to apply to the request before each execution. + +Available transforms for request: [`append`, `delete`, `set`]. + +Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`]. + +Can write state to: [`header.*`, `url.params.*`, `body.*`]. + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + is_v2: true + request.url: http://localhost:9200/_search?scroll=5m + request.method: POST + request.transforms: + - set: + target: body.from + value: '{{now (parseDuration "-1h")}}' +---- + +[float] +==== `response.transforms` + +List of transforms to apply to the response once it is received. + +Available transforms for response: [`append`, `delete`, `set`]. + +Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`, `.header.*`, `.url.*`]. + +Can write state to: [`body.*`]. + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + is_v2: true + request.url: http://localhost:9200/_search?scroll=5m + request.method: POST + response.transforms: + - delete: + target: body.very_confidential + response.split: + target: .body.hits.hits + response.pagination: + - set: + target: url.value + value: http://localhost:9200/_search/scroll + - set: + target: .url.params.scroll_id + value: '{{.last_request.body._scroll_id}}' + - set: + target: .body.scroll + value: 5m +---- + +[float] +==== `response.split` + +Split operation to apply to the response once it is received. A split can convert a map or an array into multiple events. + +[float] +==== `response.split[].target` + +Defines the target field upon the split operation will be performed. + +[float] +==== `response.split[].type` + +Defines the field type of the target. Allowed values: `array`, `map`. Default: `array`. + +[float] +==== `response.split[].transforms` + +A set of transforms can be defined. This list will be applied after `response.transforms` and after the object has been modified based on `response.split[].keep_parent` and `response.split[].key_field`. + +Available transforms for response: [`append`, `delete`, `set`]. + +Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`, `.header.*`, `.url.*`]. + +Can write state to: [`body.*`]. + +NOTE: in this context, `body.*` will be the result of all the previous transformations. + +[float] +==== `response.split[].keep_parent` + +If set to true, the fields from the parent document (at the same level as `target`) will be kept. Otherwise a new document will be created using `target` as the root. Default: `false`. + +[float] +==== `response.split[].key_field` + +It can only be used with `type: map`. When not empty, will define the a new field where the original key value will be stored. + +[float] +==== `response.split[].split` + +Nested split operation. Split operations can be nested at will, an event won't be created until the deepest split operation is applied. + +[float] +==== `response.pagination` + +List of transforms to apply to the response to every new page request. All the transforms from `request.transform` will be executed and then `response.pagination` will be added to modify the next request as needed. For subsequent responses, the usual `response.transforms` and `response.split` will be executed normally. + +Available transforms for pagination: [`append`, `delete`, `set`]. + +Can read state from: [`.last_response.*`, `.last_event.*`, `.cursor.*`]. + +Can write state to: [`body.*`, `header.*`, `url.*`]. + +Examples using split: + +- We have a response with two nested arrays and we want a document for each of the elements of the inner array: + +["source","json",subs="attributes"] +---- +{ + "this": "is kept", + "alerts": [ + { + "this_is": "also kept", + "entities": [ + { + "something": "something" + }, + { + "else": "else" + } + ] + }, + { + "this_is": "also kept 2", + "entities": [ + { + "something": "something 2" + }, + { + "else": "else 2" + } + ] + } + ] +} +---- + +Our config will look like + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + is_v2: true + interval: 1m + request.url: https://example.com + response.split: + target: body.alerts + type: array + keep_parent: true + split: + # paths in nested splits need to represent the state of body, not only their current level of nesting + target: body.alerts.entities + type: array + keep_parent: true +---- + +This will output: + +["source","json",subs="attributes"] +---- +[ + { + "this": "is kept", + "alerts": { + "this_is": "also kept", + "entities": { + "something": "something" + } + } + }, + { + "this": "is kept", + "alerts": { + "this_is": "also kept", + "entities": { + "else": "else" + } + } + }, + { + "this": "is kept", + "alerts": { + "this_is": "also kept 2", + "entities": { + "something": "something 2" + } + } + }, + { + "this": "is kept", + "alerts": { + "this_is": "also kept 2", + "entities": { + "else": "else 2" + } + } + } +] +---- + +- We have a response with two an array with objects, and we want a document for each of the object keys while keeping the keys values: + +["source","json",subs="attributes"] +---- +{ + "this": "is not kept", + "alerts": [ + { + "this_is": "kept", + "entities": { + "id1": { + "something": "something" + } + } + }, + { + "this_is": "kept 2", + "entities": { + "id2": { + "something": "something 2" + } + } + } + ] +} +---- + +Our config will look like + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + is_v2: true + interval: 1m + request.url: https://example.com + response.split: + target: body.alerts + type: array + keep_parent: false + split: + # this time alerts will not exist because previous keep_parent is false + target: body.entities + type: map + keep_parent: true + key_field: id +---- + +This will output: + +["source","json",subs="attributes"] +---- +[ + { + "this_is": "kept", + "entities": { + "id": "id1", + "something": "something" + } + }, + { + "this_is": "kept 2", + "entities": { + "id": "id2", + "something": "something 2" + } + } +] +---- + +- We have a response with two an array with objects, and we want a document for each of the object keys while applying a transform to each: + +["source","json",subs="attributes"] +---- +{ + "this": "is not kept", + "alerts": [ + { + "this_is": "also not kept", + "entities": { + "id1": { + "something": "something" + } + } + }, + { + "this_is": "also not kept", + "entities": { + "id2": { + "something": "something 2" + } + } + } + ] +} +---- + +Our config will look like + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + is_v2: true + interval: 1m + request.url: https://example.com + response.split: + target: body.alerts + type: array + split: + transforms: + - set: + target: body.new + value: will be added to each + target: body.entities + type: map +---- + +This will output: + +["source","json",subs="attributes"] +---- +[ + { + "something": "something", + "new": "will be added for each" + }, + { + "something": "something 2", + "new": "will be added for each" + } +] +---- + +[float] +==== `cursor` + +Cursor is a list of key value objects where an arbitrary values are defined. The values are interpreted as value templates and a default template can be set. Cursor state is kept between input restarts. + +Can read state from: [`.last_response.*`, `.last_event.*`]. + +NOTE: Default templates do not have access to any state, only to functions. + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + is_v2: true + interval: 1m + request.url: https://api.ipify.org/?format=json + response.transforms: + - set: + target: body.last_requested_at + value: '{{.cursor.last_requested_at}}' + default: "{{now}}" + cursor: + last_requested_at: + value: '{{now}}' + processors: + - decode_json_fields + fields: [message] + target: json +---- + +[float] +==== `api_key` + +Deprecated, use `request.transforms`. + +[float] +==== `http_client_timeout` + +Deprecated, use `request.timeout`. + +[float] +==== `http_headers` + +Deprecated, use `request.transforms`. + +[float] +==== `http_method` + +Deprecated, use `request.method`. + +[float] +==== `http_request_body` + +Deprecated, use `request.body`. + +[float] +==== `json_objects_array` + +Deprecated, use `request.split`. + +[float] +==== `split_events_by` + +Deprecated, use `request.split`. + +[float] +==== `no_http_body` + +Deprecated. + +[float] +==== `pagination.*` + +Deprecated, use `response.pagination`. + +[float] +==== `rate_limit.*` + +Deprecated, use `request.rate_limit.*`. +[float] +==== `retry.*` + +Deprecated, use `request.retry.*`. + +[float] +==== `ssl` + +Deprecated, use `request.ssl`. + +[float] +==== `url` + +Deprecated, use `request.url`. + +[float] +==== `oauth2.*` + +Deprecated, use `auth.oauth2.*`. + +==== Request life cycle + + +.... ++-------+ +-------------------+ +---------------------+ +---------------------------+ +---------------+ +---------+ +| Input | | RequestTransforms | | ResponsePagination | | ResponseTransforms/Split | | RemoteServer | | Output | ++-------+ +-------------------+ +---------------------+ +---------------------------+ +---------------+ +---------+ + | | | | | | + | 1. At interval, | | | | | + | create a new request. | | | | | + |------------------------------->| | | | | + | ---------------------------\ | | | | | + | | 2. Transform the request |-| | | | | + | | before executing it. | | | | | | + | |--------------------------| | | | | | + | | 3. Execute request. | | | | + | |--------------------------------------------------------------------------------------->| | + | | | | | | + | | | | 4. Return response. | | + | | | |<---------------------------| | + | | | -------------------------\ | | | + | | | | 5. Transform response |-| | | + | | | | into a list of events. | | | | + | | | |------------------------| | | | + | | | | 6. Publish every | | + | | | | event to output. | | + | | | |------------------------------------------>| + | | | -----------------------------\ | | | + | | |-| 7. If there are more pages | | | | + | | | | transform the request. | | | | + | | | |----------------------------| | | | + | | | | | | + | | | Execute request and go back to 4. | | | + | | |---------------------------------------------------------------->| | +.... + +. At every defined interval a new request will be created. +. The request will be transformed using the configured `request.transforms`. +. The resulting transformed request will be executed. +. The server will respond (here is where any retry or rate limit policy will take place when configured). +. The response will be transformed using the configured `response.transforms` and `response.split`. +. Each resulting event will be published to the output. +. If a `response.pagination` is configured and there are more pages, a new request will be created using it, otherwise the process ends until the next interval. + [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 9a29bc76962..390305dd34b 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -216,7 +216,7 @@ processors: # At debug level, you can selectively enable logging only for some components. # To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publish", "service". +# "publisher", "service". #logging.selectors: ["*"] # ============================= X-Pack Monitoring ============================== diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config.go b/x-pack/filebeat/input/httpjson/internal/v2/config.go index 893d17ccfad..0f532e6ee8d 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config.go @@ -19,7 +19,7 @@ type config struct { type cursorConfig map[string]struct { Value *valueTpl `config:"value"` - Default string `config:"default"` + Default *valueTpl `config:"default"` } func (c config) Validate() error { @@ -31,14 +31,21 @@ func (c config) Validate() error { func defaultConfig() config { timeout := 30 * time.Second + maxAttempts := 5 + waitMin := time.Second + waitMax := time.Minute return config{ Interval: time.Minute, Auth: &authConfig{}, Request: &requestConfig{ - Timeout: &timeout, - Method: "GET", - RedirectHeadersForward: true, - RedirectLocationTrusted: false, + Timeout: &timeout, + Method: "GET", + Retry: retryConfig{ + MaxAttempts: &maxAttempts, + WaitMin: &waitMin, + WaitMax: &waitMax, + }, + RedirectForwardHeaders: true, RedirectHeadersBanList: []string{ "WWW-Authenticate", "Authorization", diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go index 2978f42cc47..a76b115cfca 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go @@ -76,18 +76,17 @@ func (u *urlConfig) Unpack(in string) error { } type requestConfig struct { - URL *urlConfig `config:"url" validate:"required"` - Method string `config:"method" validate:"required"` - Body *common.MapStr `config:"body"` - Timeout *time.Duration `config:"timeout"` - SSL *tlscommon.Config `config:"ssl"` - Retry retryConfig `config:"retry"` - RedirectHeadersForward bool `config:"redirect.headers.forward"` - RedirectHeadersBanList []string `config:"redirect.headers.ban_list"` - RedirectLocationTrusted bool `config:"redirect.location_trusted"` - RedirectMaxRedirects int `config:"redirect.max_redirects"` - RateLimit *rateLimitConfig `config:"rate_limit"` - Transforms transformsConfig `config:"transforms"` + URL *urlConfig `config:"url" validate:"required"` + Method string `config:"method" validate:"required"` + Body *common.MapStr `config:"body"` + Timeout *time.Duration `config:"timeout"` + SSL *tlscommon.Config `config:"ssl"` + Retry retryConfig `config:"retry"` + RedirectForwardHeaders bool `config:"redirect.forward_headers"` + RedirectHeadersBanList []string `config:"redirect.headers_ban_list"` + RedirectMaxRedirects int `config:"redirect.max_redirects"` + RateLimit *rateLimitConfig `config:"rate_limit"` + Transforms transformsConfig `config:"transforms"` } func (c requestConfig) getTimeout() time.Duration { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go index 11e6eb8be8a..1e3e8132402 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -13,7 +13,7 @@ import ( "net/url" "time" - "github.com/hashicorp/go-retryablehttp" + retryablehttp "github.com/hashicorp/go-retryablehttp" "go.uber.org/zap" v2 "github.com/elastic/beats/v7/filebeat/input/v2" @@ -184,7 +184,7 @@ func checkRedirect(config *requestConfig) func(*http.Request, []*http.Request) e return fmt.Errorf("stopped after %d redirects", config.RedirectMaxRedirects) } - if !config.RedirectHeadersForward || len(via) == 0 { + if !config.RedirectForwardHeaders || len(via) == 0 { return nil } @@ -192,10 +192,8 @@ func checkRedirect(config *requestConfig) func(*http.Request, []*http.Request) e req.Header = prev.Header.Clone() - if !config.RedirectLocationTrusted { - for _, k := range config.RedirectHeadersBanList { - req.Header.Del(k) - } + for _, k := range config.RedirectHeadersBanList { + req.Header.Del(k) } return nil diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go index 2e8fb9b01d9..29926ffff2a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go @@ -104,7 +104,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { tr := emptyTransformable() tr.header = resp.Header - remaining := r.remaining.Execute(emptyTransformContext(), tr, "", r.log) + remaining := r.remaining.Execute(emptyTransformContext(), tr, nil, r.log) if remaining == "" { return 0, errors.New("remaining value is empty") } @@ -122,7 +122,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { return 0, nil } - reset := r.reset.Execute(emptyTransformContext(), tr, "", r.log) + reset := r.reset.Execute(emptyTransformContext(), tr, nil, r.log) if reset == "" { return 0, errors.New("reset value is empty") } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go index 8748f9e4efa..c5510d832c5 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split.go @@ -71,13 +71,6 @@ func newSplit(c *splitConfig, log *logp.Logger) (*split, error) { func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMsg) error { respCpy := resp.clone() - var err error - for _, t := range s.transforms { - respCpy, err = t.run(ctx, respCpy) - if err != nil { - return err - } - } v, err := respCpy.body.GetValue(s.targetInfo.Name) if err != nil && err != common.ErrKeyNotFound { @@ -157,6 +150,14 @@ func (s *split) sendEvent(ctx transformContext, resp *transformable, key string, resp.body = m } + var err error + for _, t := range s.transforms { + resp, err = t.run(ctx, resp) + if err != nil { + return err + } + } + if s.split != nil { return s.split.run(ctx, resp, ch) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go index 0e42ba91f1a..c19f559d944 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go @@ -14,6 +14,8 @@ import ( ) func TestSplit(t *testing.T) { + registerResponseTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) cases := []struct { name string config *splitConfig @@ -139,6 +141,60 @@ func TestSplit(t *testing.T) { }, expectedErr: nil, }, + { + name: "A nested array with a nested map with transforms", + config: &splitConfig{ + Target: "body.alerts", + Type: "array", + Split: &splitConfig{ + Target: "body.entities", + Type: "map", + Transforms: transformsConfig{ + common.MustNewConfigFrom(map[string]interface{}{ + "set": map[string]interface{}{ + "target": "body.foo", + "value": "set for each", + }, + }), + }, + }, + }, + ctx: emptyTransformContext(), + resp: &transformable{ + body: common.MapStr{ + "this": "is not kept", + "alerts": []interface{}{ + map[string]interface{}{ + "this_is": "kept", + "entities": map[string]interface{}{ + "id1": map[string]interface{}{ + "something": "else", + }, + }, + }, + map[string]interface{}{ + "this_is": "also not kept", + "entities": map[string]interface{}{ + "id2": map[string]interface{}{ + "something": "else 2", + }, + }, + }, + }, + }, + }, + expectedMessages: []common.MapStr{ + { + "something": "else", + "foo": "set for each", + }, + { + "something": "else 2", + "foo": "set for each", + }, + }, + expectedErr: nil, + }, } for _, tc := range cases { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go index f264f6a22af..68cf41bbaae 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go @@ -18,14 +18,14 @@ const appendName = "append" type appendConfig struct { Target string `config:"target"` Value *valueTpl `config:"value"` - Default string `config:"default"` + Default *valueTpl `config:"default"` } type appendt struct { log *logp.Logger targetInfo targetInfo value *valueTpl - defaultValue string + defaultValue *valueTpl runFunc func(ctx transformContext, transformable *transformable, key, val string) error } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go index 691885fc5b0..ed288ac6356 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go @@ -21,14 +21,14 @@ const setName = "set" type setConfig struct { Target string `config:"target"` Value *valueTpl `config:"value"` - Default string `config:"default"` + Default *valueTpl `config:"default"` } type set struct { log *logp.Logger targetInfo targetInfo value *valueTpl - defaultValue string + defaultValue *valueTpl runFunc func(ctx transformContext, transformable *transformable, key, val string) error } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go index a102c3b9bd8..4209f1e0ac2 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go @@ -8,9 +8,10 @@ import ( "net/http" "testing" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" - "github.com/stretchr/testify/assert" ) func TestEmptyTransformContext(t *testing.T) { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index c6bac668141..cf38afa596b 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -24,9 +24,9 @@ func (t *valueTpl) Unpack(in string) error { Option("missingkey=error"). Funcs(template.FuncMap{ "now": now, - "hour": hour, "parseDate": parseDate, "formatDate": formatDate, + "parseDuration": parseDuration, "parseTimestamp": parseTimestamp, "parseTimestampMilli": parseTimestampMilli, "parseTimestampNano": parseTimestampNano, @@ -42,12 +42,14 @@ func (t *valueTpl) Unpack(in string) error { return nil } -func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal string, log *logp.Logger) (val string) { +func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal *valueTpl, log *logp.Logger) (val string) { defer func() { if r := recover(); r != nil { err := r.(error) log.Infof("template execution: %v", err) - val = defaultVal + if defaultVal != nil { + val = defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) + } } }() @@ -67,12 +69,12 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal if err := t.Template.Execute(buf, data); err != nil { log.Infof("template execution: %v", err) - return defaultVal + return defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) } val = buf.String() if val == "" || strings.Contains(val, "") { - val = defaultVal + val = defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) } return val } @@ -94,15 +96,16 @@ var ( ) func now(add ...time.Duration) time.Time { - now := timeNow() + now := timeNow().UTC() if len(add) == 0 { return now } return now.Add(add[0]) } -func hour(n int) time.Duration { - return time.Duration(n) * time.Hour +func parseDuration(s string) time.Duration { + d, _ := time.ParseDuration(s) + return d } func parseDate(date string, layout ...string) time.Time { @@ -121,7 +124,7 @@ func parseDate(date string, layout ...string) time.Time { return time.Time{} } - return t + return t.UTC() } func formatDate(date time.Time, layouttz ...string) string { @@ -149,15 +152,15 @@ func formatDate(date time.Time, layouttz ...string) string { } func parseTimestamp(s int64) time.Time { - return time.Unix(s, 0) + return time.Unix(s, 0).UTC() } func parseTimestampMilli(ms int64) time.Time { - return time.Unix(0, ms*1e6) + return time.Unix(0, ms*1e6).UTC() } func parseTimestampNano(ns int64) time.Time { - return time.Unix(0, ns) + return time.Unix(0, ns).UTC() } var regexpLinkRel = regexp.MustCompile(`<(.*)>;.*\srel\="?([^;"]*)`) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 9a8b2e37be9..33dc1d5f0b5 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -41,6 +41,7 @@ func TestValueTpl(t *testing.T) { name: "can render default value if execute fails", value: "{{.last_response.body.does_not_exist}}", paramCtx: transformContext{ + lastEvent: &common.MapStr{}, lastResponse: emptyTransformable(), }, paramTr: emptyTransformable(), @@ -62,8 +63,8 @@ func TestValueTpl(t *testing.T) { expected: "25", }, { - name: "func hour", - value: `{{ hour -1 }}`, + name: "func parseDuration", + value: `{{ parseDuration "-1h" }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "-1h0m0s", @@ -81,7 +82,7 @@ func TestValueTpl(t *testing.T) { name: "func now with duration", setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, teardown: func() { timeNow = time.Now }, - value: `{{ now (-1|hour) }}`, + value: `{{ now (parseDuration "-1h") }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "2020-11-05 12:25:32 +0000 UTC", @@ -102,7 +103,7 @@ func TestValueTpl(t *testing.T) { }, { name: "func parseDate with custom layout", - value: `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC }}`, + value: `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006") }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "2020-11-05 12:25:32 +0000 UTC", @@ -136,21 +137,21 @@ func TestValueTpl(t *testing.T) { }, { name: "func parseTimestamp", - value: `{{ (parseTimestamp 1604582732).UTC }}`, + value: `{{ (parseTimestamp 1604582732) }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "2020-11-05 13:25:32 +0000 UTC", }, { name: "func parseTimestampMilli", - value: `{{ (parseTimestampMilli 1604582732000).UTC }}`, + value: `{{ (parseTimestampMilli 1604582732000) }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "2020-11-05 13:25:32 +0000 UTC", }, { name: "func parseTimestampNano", - value: `{{ (parseTimestampNano 1604582732000000000).UTC }}`, + value: `{{ (parseTimestampNano 1604582732000000000) }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "2020-11-05 13:25:32 +0000 UTC", @@ -200,7 +201,7 @@ func TestValueTpl(t *testing.T) { name: "can execute functions pipeline", setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, teardown: func() { timeNow = time.Now }, - value: `{{ -1 | hour | now | formatDate }}`, + value: `{{ (parseDuration "-1h") | now | formatDate }}`, paramCtx: emptyTransformContext(), paramTr: emptyTransformable(), expected: "2020-11-05T12:25:32Z", @@ -218,7 +219,9 @@ func TestValueTpl(t *testing.T) { } tpl := &valueTpl{} assert.NoError(t, tpl.Unpack(tc.value)) - got := tpl.Execute(tc.paramCtx, tc.paramTr, tc.paramDefVal, logp.NewLogger("")) + defTpl := &valueTpl{} + assert.NoError(t, defTpl.Unpack(tc.paramDefVal)) + got := tpl.Execute(tc.paramCtx, tc.paramTr, defTpl, logp.NewLogger("")) assert.Equal(t, tc.expected, got) }) } From bdbb877dc0016d342141e4cf735694eafd3e87a2 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 26 Nov 2020 09:00:28 +0100 Subject: [PATCH 26/35] Add more debug logs and split tests --- .../input/httpjson/internal/v2/cursor.go | 1 + .../input/httpjson/internal/v2/split.go | 11 ++- .../input/httpjson/internal/v2/split_test.go | 83 +++++++++++++++++++ .../input/httpjson/internal/v2/value_tpl.go | 22 +++-- 4 files changed, 107 insertions(+), 10 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go b/x-pack/filebeat/input/httpjson/internal/v2/cursor.go index ca555b80777..bb02daecf50 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/cursor.go @@ -24,6 +24,7 @@ func newCursor(cfg cursorConfig, log *logp.Logger) *cursor { func (c *cursor) load(cursor *inputcursor.Cursor) { if c == nil || cursor == nil || cursor.IsNew() { + c.log.Debug("new cursor: nothing loaded") return } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go index c5510d832c5..93c921b28e9 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split.go @@ -85,7 +85,9 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMs } if len(arr) == 0 { - return errEmtpyField + if err := s.sendEvent(ctx, respCpy, "", nil, ch); err != nil { + return err + } } for _, a := range arr { @@ -106,7 +108,9 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMs } if len(ms) == 0 { - return errEmtpyField + if err := s.sendEvent(ctx, respCpy, "", nil, ch); err != nil { + return err + } } for k, v := range ms { @@ -123,6 +127,9 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMs func toMapStr(v interface{}) (common.MapStr, bool) { var m common.MapStr + if v == nil { + return m, true + } switch ts := v.(type) { case common.MapStr: m = ts diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go index c19f559d944..4fb0eca1e20 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go @@ -195,6 +195,89 @@ func TestSplit(t *testing.T) { }, expectedErr: nil, }, + { + name: "A nested array with a nested array in an object", + config: &splitConfig{ + Target: "body.response", + Type: "array", + Split: &splitConfig{ + Target: "body.Event.Attributes", + KeepParent: true, + }, + }, + ctx: emptyTransformContext(), + resp: &transformable{ + body: common.MapStr{ + "response": []interface{}{ + map[string]interface{}{ + "Event": map[string]interface{}{ + "timestamp": "1606324417", + "Attributes": []interface{}{ + map[string]interface{}{ + "key": "value", + }, + map[string]interface{}{ + "key2": "value2", + }, + }, + }, + }, + }, + }, + }, + expectedMessages: []common.MapStr{ + { + "Event": common.MapStr{ + "timestamp": "1606324417", + "Attributes": common.MapStr{ + "key": "value", + }, + }, + }, + { + "Event": common.MapStr{ + "timestamp": "1606324417", + "Attributes": common.MapStr{ + "key2": "value2", + }, + }, + }, + }, + expectedErr: nil, + }, + { + name: "A nested array with an empty nested array in an object", + config: &splitConfig{ + Target: "body.response", + Type: "array", + Split: &splitConfig{ + Target: "body.Event.Attributes", + KeepParent: true, + }, + }, + ctx: emptyTransformContext(), + resp: &transformable{ + body: common.MapStr{ + "response": []interface{}{ + map[string]interface{}{ + "Event": map[string]interface{}{ + "timestamp": "1606324417", + "Attributes": []interface{}{}, + }, + }, + }, + }, + }, + expectedMessages: []common.MapStr{ + { + "Event": common.MapStr{ + "timestamp": "1606324417", + "Attributes": common.MapStr{}, + }, + }, + }, + expectedErr: nil, + }, } for _, tc := range cases { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index cf38afa596b..b34c3d792e4 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -43,13 +43,20 @@ func (t *valueTpl) Unpack(in string) error { } func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal *valueTpl, log *logp.Logger) (val string) { + fallback := func(err error) string { + if err != nil { + log.Debugf("template execution failed: %v", err) + } + if defaultVal != nil { + log.Debugf("template execution: falling back to default value") + return defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) + } + return "" + } + defer func() { if r := recover(); r != nil { - err := r.(error) - log.Infof("template execution: %v", err) - if defaultVal != nil { - val = defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) - } + val = fallback(r.(error)) } }() @@ -68,13 +75,12 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal _, _ = data.Put("last_response.url.params", trCtx.lastResponse.url.Query()) if err := t.Template.Execute(buf, data); err != nil { - log.Infof("template execution: %v", err) - return defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) + return fallback(err) } val = buf.String() if val == "" || strings.Contains(val, "") { - val = defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) + return fallback(nil) } return val } From ce233a95c810dac4c6fdf817d4669ef13a94b589 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 26 Nov 2020 09:20:45 +0100 Subject: [PATCH 27/35] Ignore empty values on set and append --- x-pack/filebeat/input/httpjson/internal/v2/request.go | 6 ++++-- .../input/httpjson/internal/v2/transform_append.go | 9 +++++++++ .../filebeat/input/httpjson/internal/v2/transform_set.go | 9 +++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index c425f0aa95d..94b3117b820 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -46,7 +46,7 @@ func (c *httpClient) do(stdCtx context.Context, trCtx transformContext, req *htt return resp, nil } -func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []basicTransform) (*transformable, error) { +func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []basicTransform, log *logp.Logger) (*transformable, error) { req := emptyTransformable() req.url = url @@ -62,6 +62,8 @@ func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []ba } } + log.Debugf("new request: %#v", req) + return req, nil } @@ -93,7 +95,7 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. } func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transformContext) (*http.Request, error) { - trReq, err := newRequest(trCtx, rf.body, rf.url, rf.transforms) + trReq, err := newRequest(trCtx, rf.body, rf.url, rf.transforms, rf.log) if err != nil { return nil, err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go index 68cf41bbaae..5455d5d7403 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go @@ -116,6 +116,9 @@ func (append *appendt) run(ctx transformContext, transformable *transformable) ( } func appendToCommonMap(m common.MapStr, key, val string) error { + if val == "" { + return nil + } var value interface{} = val if found, _ := m.HasKey(key); found { prev, _ := m.GetValue(key) @@ -140,11 +143,17 @@ func appendBody(ctx transformContext, transformable *transformable, key, value s } func appendHeader(ctx transformContext, transformable *transformable, key, value string) error { + if value == "" { + return nil + } transformable.header.Add(key, value) return nil } func appendURLParams(ctx transformContext, transformable *transformable, key, value string) error { + if value == "" { + return nil + } q := transformable.url.Query() q.Add(key, value) transformable.url.RawQuery = q.Encode() diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go index ed288ac6356..14bccdfb0e0 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go @@ -121,6 +121,9 @@ func (set *set) run(ctx transformContext, transformable *transformable) (*transf } func setToCommonMap(m common.MapStr, key, val string) error { + if val == "" { + return nil + } if _, err := m.Put(key, val); err != nil { return err } @@ -132,11 +135,17 @@ func setBody(ctx transformContext, transformable *transformable, key, value stri } func setHeader(ctx transformContext, transformable *transformable, key, value string) error { + if value == "" { + return nil + } transformable.header.Add(key, value) return nil } func setURLParams(ctx transformContext, transformable *transformable, key, value string) error { + if value == "" { + return nil + } q := transformable.url.Query() q.Set(key, value) transformable.url.RawQuery = q.Encode() From 5f685fadb75729fced8fcb1f155dd224d2d57400 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 26 Nov 2020 11:03:55 +0100 Subject: [PATCH 28/35] Allow content type and accept override and change redirect default --- .../docs/inputs/input-httpjson.asciidoc | 4 +-- .../input/httpjson/internal/v2/config.go | 8 ++---- .../input/httpjson/internal/v2/input.go | 9 +++++-- .../input/httpjson/internal/v2/request.go | 27 ++++++++++--------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index e914b990bee..cb43e033510 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -385,12 +385,12 @@ This specifies the maximum time to wait before a retry is attempted. Default: `6 [float] ==== `request.redirect.forward_headers` -This specifies if headers are forwarded in case of a redirect. Default: `true`. +This specifies if headers are forwarded in case of a redirect. Default: `false`. [float] ==== `request.redirect.headers_ban_list` -When `redirect.forward_headers` is set to `true`, all headers __except__ the ones defined in this list, will be forwarded. Default: `["WWW-Authenticate", "Authorization"]`. +When `redirect.forward_headers` is set to `true`, all headers __except__ the ones defined in this list, will be forwarded. Default: `[]`. [float] ==== `request.redirect.max_redirects` diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config.go b/x-pack/filebeat/input/httpjson/internal/v2/config.go index 0f532e6ee8d..95eac252201 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config.go @@ -45,12 +45,8 @@ func defaultConfig() config { WaitMin: &waitMin, WaitMax: &waitMax, }, - RedirectForwardHeaders: true, - RedirectHeadersBanList: []string{ - "WWW-Authenticate", - "Authorization", - }, - RedirectMaxRedirects: 10, + RedirectForwardHeaders: false, + RedirectMaxRedirects: 10, }, Response: &responseConfig{}, } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go index 1e3e8132402..f63b3769436 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -155,7 +155,7 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC DisableKeepAlives: true, }, Timeout: timeout, - CheckRedirect: checkRedirect(config.Request), + CheckRedirect: checkRedirect(config.Request, log), }, Logger: newRetryLogger(), RetryWaitMin: config.Request.Retry.getWaitMin(), @@ -178,21 +178,26 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC return &httpClient{client: client.StandardClient(), limiter: limiter}, nil } -func checkRedirect(config *requestConfig) func(*http.Request, []*http.Request) error { +func checkRedirect(config *requestConfig, log *logp.Logger) func(*http.Request, []*http.Request) error { return func(req *http.Request, via []*http.Request) error { + log.Debug("http client: checking redirect") if len(via) >= config.RedirectMaxRedirects { + log.Debug("http client: max redirects exceeded") return fmt.Errorf("stopped after %d redirects", config.RedirectMaxRedirects) } if !config.RedirectForwardHeaders || len(via) == 0 { + log.Debugf("http client: nothing to do while checking redirects - forward_headers: %v, via: %#v", config.RedirectForwardHeaders, via) return nil } prev := via[len(via)-1] // previous request to get headers from + log.Debugf("http client: forwarding headers from previous request: %#v", prev.Header) req.Header = prev.Header.Clone() for _, k := range config.RedirectHeadersBanList { + log.Debugf("http client: ban header %v", k) req.Header.Del(k) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index 94b3117b820..d17ce445e25 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -46,23 +46,29 @@ func (c *httpClient) do(stdCtx context.Context, trCtx transformContext, req *htt return resp, nil } -func newRequest(ctx transformContext, body *common.MapStr, url url.URL, trs []basicTransform, log *logp.Logger) (*transformable, error) { +func (rf *requestFactory) newRequest(ctx transformContext) (*transformable, error) { req := emptyTransformable() - req.url = url + req.url = rf.url - if body != nil { - req.body.DeepUpdate(*body) + if rf.body != nil { + req.body.DeepUpdate(*rf.body) + } + + req.header.Set("Accept", "application/json") + req.header.Set("User-Agent", userAgent) + if rf.method == "POST" { + req.header.Set("Content-Type", "application/json") } var err error - for _, t := range trs { + for _, t := range rf.transforms { req, err = t.run(ctx, req) if err != nil { return nil, err } } - log.Debugf("new request: %#v", req) + rf.log.Debugf("new request: %#v", req) return req, nil } @@ -95,7 +101,7 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. } func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transformContext) (*http.Request, error) { - trReq, err := newRequest(trCtx, rf.body, rf.url, rf.transforms, rf.log) + trReq, err := rf.newRequest(trCtx) if err != nil { return nil, err } @@ -120,12 +126,7 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transform req = req.WithContext(stdCtx) - req.Header = trReq.header - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", userAgent) - if rf.method == "POST" { - req.Header.Set("Content-Type", "application/json") - } + req.Header = trReq.header.Clone() if rf.user != "" || rf.password != "" { req.SetBasicAuth(rf.user, rf.password) From 55c02754f0ce5540ac67615ad525413528db60cb Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 26 Nov 2020 13:22:42 +0100 Subject: [PATCH 29/35] Add add and toInt functions --- .../docs/inputs/input-httpjson.asciidoc | 5 ++- .../httpjson/internal/v2/config_response.go | 2 +- .../input/httpjson/internal/v2/pagination.go | 24 +++++----- .../input/httpjson/internal/v2/request.go | 2 +- .../input/httpjson/internal/v2/response.go | 16 ++++--- .../input/httpjson/internal/v2/split.go | 45 ++++++++++++++----- .../input/httpjson/internal/v2/transform.go | 2 + .../input/httpjson/internal/v2/value_tpl.go | 18 ++++++++ 8 files changed, 83 insertions(+), 31 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index cb43e033510..dbcdfbf00eb 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -90,6 +90,7 @@ The state has the following elements: - `last_request.url.params`: A map containing the params from the URL in `last_response.url.value`. - `last_response.header`: A map containing the headers from the last successful response. - `last_response.body`: A map containing the parsed JSON body from the last successful response. (before any transform) +- `last_response.page`: A number indicating the page number of the last response. - `last_event`: A map representing the last event sent to the output (result from applying transforms to `last_response.body`). - `url.value`: The full URL with params and fragments. Will make reference to the request URL for configs under `request.*` and for the last successful request under `response.*`. - `url.params`: A map containing the URL params. Will make reference to the request URL for configs under `request.*` and for the last successful request under `response.*`. @@ -181,6 +182,8 @@ Some built in helper functions are provided to work with the input state inside - `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Go lang predefined layouts or a custom one. Example: `{{ parseDate "2020-11-05T12:25:32Z" }}`, `{{ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" }}`, `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC }}`. - `formatDate`: formats a `time.Time`. By default the format layout is `RFC3339` but optionally can accept any of the Go lang predefined layouts or a custom one. It will default to UTC timezone when formatting but optionally a different timezone can be specified. If the timezone is incorrect will default to UTC. Example: `{{ formatDate (now) "UnixDate" }}`, `{{ formatDate (now) "UnixDate" "America/New_York" }}`. - `getRFC5988Link`: it extracts a specific relation from a list of https://tools.ietf.org/html/rfc5988[RFC5988] links. It is useful when we are parsing header values for pagination, for example. Example: `{{ getRFC5988Link "next" .last_response.header.Link }}` +- `toInt`: converts a string to an integer, returns 0 if it fails. +- `add`: adds a list of integers and returns their sum. In addition to the provided functions, any of the native functions for `time.Time` and `http.Header` types can be used on the corresponding objects. Examples: `{{(now).Day}}`, `{{.last_response.header.Get "key"}}` @@ -761,7 +764,7 @@ This will output: [float] ==== `cursor` -Cursor is a list of key value objects where an arbitrary values are defined. The values are interpreted as value templates and a default template can be set. Cursor state is kept between input restarts. +Cursor is a list of key value objects where an arbitrary values are defined. The values are interpreted as value templates and a default template can be set. Cursor state is kept between input restarts and updated once all the events for a request are published. Can read state from: [`.last_response.*`, `.last_event.*`]. diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go index 55b45650a2d..6b616e79d30 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go @@ -33,7 +33,7 @@ func (c *responseConfig) Validate() error { if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace, nil); err != nil { return err } - if _, err := newBasicTransformsFromConfig(c.Transforms, paginationNamespace, nil); err != nil { + if _, err := newBasicTransformsFromConfig(c.Pagination, paginationNamespace, nil); err != nil { return err } return nil diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index a49e1d3800c..5ba339afdcd 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -74,6 +74,8 @@ type pageIterator struct { isFirst bool done bool + + n int } func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformContext, resp *http.Response) *pageIterator { @@ -86,9 +88,9 @@ func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformCont } } -func (iter *pageIterator) next() (*transformable, bool, error) { +func (iter *pageIterator) next() (*transformable, int, bool, error) { if iter == nil || iter.resp == nil || iter.done { - return nil, false, nil + return nil, 0, false, nil } if iter.isFirst { @@ -96,13 +98,13 @@ func (iter *pageIterator) next() (*transformable, bool, error) { iter.isFirst = false tr, err := iter.getPage() if err != nil { - return nil, false, err + return nil, 0, false, err } if iter.pagination.requestFactory == nil { iter.pagination.log.Debug("last page") iter.done = true } - return tr, true, nil + return tr, iter.n, true, nil } httpReq, err := iter.pagination.requestFactory.newHTTPRequest(iter.stdCtx, iter.trCtx) @@ -112,30 +114,30 @@ func (iter *pageIterator) next() (*transformable, bool, error) { // did not find any new value and we can stop paginating without error iter.pagination.log.Debug("last page") iter.done = true - return nil, false, nil + return nil, 0, false, nil } - return nil, false, err + return nil, 0, false, err } resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) if err != nil { - return nil, false, err + return nil, 0, false, err } iter.resp = resp tr, err := iter.getPage() if err != nil { - return nil, false, err + return nil, 0, false, err } if len(tr.body) == 0 { iter.pagination.log.Debug("finished pagination because there is no body") iter.done = true - return nil, false, nil + return nil, 0, false, nil } - return tr, true, nil + return tr, iter.n, true, nil } func (iter *pageIterator) getPage() (*transformable, error) { @@ -155,5 +157,7 @@ func (iter *pageIterator) getPage() (*transformable, error) { } } + iter.n += 1 + return tr, nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index d17ce445e25..a52964244df 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -191,9 +191,9 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu } *trCtx.lastEvent = maybeMsg.msg - trCtx.cursor.update(trCtx) n += 1 } + trCtx.cursor.update(trCtx) r.log.Infof("request finished: %d events published", n) return nil diff --git a/x-pack/filebeat/input/httpjson/internal/v2/response.go b/x-pack/filebeat/input/httpjson/internal/v2/response.go index c98366ef434..9f98720376e 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/response.go @@ -52,7 +52,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans iter := rp.pagination.newPageIterator(stdCtx, trCtx, resp) for { - page, hasNext, err := iter.next() + page, pageN, hasNext, err := iter.next() if err != nil { ch <- maybeMsg{err: err} return @@ -62,13 +62,15 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans return } + *trCtx.lastPage = pageN *trCtx.lastResponse = *page.clone() - rp.log.Debugf("last received page: %v", trCtx.lastResponse) + rp.log.Debugf("last received page: %#v", trCtx.lastResponse) for _, t := range rp.transforms { page, err = t.run(trCtx, page) if err != nil { + rp.log.Debug("error transforming page") ch <- maybeMsg{err: err} return } @@ -76,15 +78,17 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans if rp.split == nil { ch <- maybeMsg{msg: page.body} + rp.log.Debug("no split found: continuing to next page") continue } if err := rp.split.run(trCtx, page, ch); err != nil { - if err == errEmtpyField { - // nothing else to send - return + if err == errEmptyField { + // nothing else to send for this page + rp.log.Debug("split operation finished") + continue } - + rp.log.Debug("split operation failed") ch <- maybeMsg{err: err} return } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go index 93c921b28e9..80a21bb4d40 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split.go @@ -12,9 +12,13 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -var errEmtpyField = errors.New("the requested field is emtpy") +var ( + errEmptyField = errors.New("the requested field is emtpy") + errEmptyTopField = errors.New("the requested top split field is emtpy") +) type split struct { + log *logp.Logger targetInfo targetInfo kind string transforms []basicTransform @@ -60,6 +64,7 @@ func newSplit(c *splitConfig, log *logp.Logger) (*split, error) { } return &split{ + log: log, targetInfo: ti, kind: c.Type, keepParent: c.KeepParent, @@ -70,6 +75,10 @@ func newSplit(c *splitConfig, log *logp.Logger) (*split, error) { } func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMsg) error { + return s.runChild(ctx, resp, ch, false) +} + +func (s *split) runChild(ctx transformContext, resp *transformable, ch chan<- maybeMsg, isChild bool) error { respCpy := resp.clone() v, err := respCpy.body.GetValue(s.targetInfo.Name) @@ -79,19 +88,23 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMs switch s.kind { case "", splitTypeArr: + if v == nil { + s.log.Debug("array field is nil, sending main body") + return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) + } + arr, ok := v.([]interface{}) if !ok { return fmt.Errorf("field %s needs to be an array to be able to split on it but it is %T", s.targetInfo.Name, v) } if len(arr) == 0 { - if err := s.sendEvent(ctx, respCpy, "", nil, ch); err != nil { - return err - } + s.log.Debug("array field is empty, sending main body") + return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) } for _, a := range arr { - if err := s.sendEvent(ctx, respCpy, "", a, ch); err != nil { + if err := s.sendEvent(ctx, respCpy, "", a, ch, isChild); err != nil { return err } } @@ -99,7 +112,8 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMs return nil case splitTypeMap: if v == nil { - return errEmtpyField + s.log.Debug("object field is nil, sending main body") + return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) } ms, ok := toMapStr(v) @@ -108,13 +122,12 @@ func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMs } if len(ms) == 0 { - if err := s.sendEvent(ctx, respCpy, "", nil, ch); err != nil { - return err - } + s.log.Debug("object field is empty, sending main body") + return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) } for k, v := range ms { - if err := s.sendEvent(ctx, respCpy, k, v, ch); err != nil { + if err := s.sendEvent(ctx, respCpy, k, v, ch, isChild); err != nil { return err } } @@ -141,7 +154,11 @@ func toMapStr(v interface{}) (common.MapStr, bool) { return m, true } -func (s *split) sendEvent(ctx transformContext, resp *transformable, key string, val interface{}, ch chan<- maybeMsg) error { +func (s *split) sendEvent(ctx transformContext, resp *transformable, key string, val interface{}, ch chan<- maybeMsg, isChild bool) error { + if val == nil && !isChild && !s.keepParent { + return errEmptyTopField + } + m, ok := toMapStr(val) if !ok { return errors.New("split can only be applied on object lists") @@ -166,10 +183,14 @@ func (s *split) sendEvent(ctx transformContext, resp *transformable, key string, } if s.split != nil { - return s.split.run(ctx, resp, ch) + return s.split.runChild(ctx, resp, ch, true) } ch <- maybeMsg{msg: resp.body.Clone()} + if val == nil { + return errEmptyField + } + return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform.go b/x-pack/filebeat/input/httpjson/internal/v2/transform.go index 18c0da78d70..fa807728ccc 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform.go @@ -24,6 +24,7 @@ type transforms []transform type transformContext struct { cursor *cursor lastEvent *common.MapStr + lastPage *int lastResponse *transformable } @@ -31,6 +32,7 @@ func emptyTransformContext() transformContext { return transformContext{ cursor: &cursor{}, lastEvent: &common.MapStr{}, + lastPage: new(int), lastResponse: emptyTransformable(), } } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index b34c3d792e4..530f0061067 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -7,6 +7,7 @@ package v2 import ( "bytes" "regexp" + "strconv" "strings" "text/template" "time" @@ -31,6 +32,8 @@ func (t *valueTpl) Unpack(in string) error { "parseTimestampMilli": parseTimestampMilli, "parseTimestampNano": parseTimestampNano, "getRFC5988Link": getRFC5988Link, + "toInt": toInt, + "add": add, }). Parse(in) if err != nil { @@ -58,6 +61,7 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal if r := recover(); r != nil { val = fallback(r.(error)) } + log.Debugf("template execution: evaluated template %q", val) }() buf := new(bytes.Buffer) @@ -69,6 +73,7 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal _, _ = data.Put("url.params", tr.url.Query()) _, _ = data.Put("cursor", trCtx.cursor.clone()) _, _ = data.Put("last_event", trCtx.lastEvent.Clone()) + _, _ = data.Put("last_response.page", trCtx.lastPage) _, _ = data.Put("last_response.body", trCtx.lastResponse.body.Clone()) _, _ = data.Put("last_response.header", trCtx.lastResponse.header.Clone()) _, _ = data.Put("last_response.url.value", trCtx.lastResponse.url.String()) @@ -191,3 +196,16 @@ func getRFC5988Link(rel string, links []string) string { return "" } + +func toInt(s string) int { + i, _ := strconv.ParseInt(s, 10, 64) + return int(i) +} + +func add(vs ...int) int { + var sum int + for _, v := range vs { + sum += v + } + return sum +} From 94082390430a4b53736cb59660054ea0a45236e4 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 27 Nov 2020 08:39:59 +0100 Subject: [PATCH 30/35] Support array responses --- .../input/httpjson/internal/v2/pagination.go | 40 ++++--- .../httpjson/internal/v2/rate_limiter_test.go | 4 + .../input/httpjson/internal/v2/response.go | 106 ++++++++++++++---- .../input/httpjson/internal/v2/split.go | 4 +- .../input/httpjson/internal/v2/split_test.go | 23 +++- .../input/httpjson/internal/v2/transform.go | 15 ++- .../httpjson/internal/v2/transform_test.go | 2 +- .../input/httpjson/internal/v2/value_tpl.go | 14 +-- .../httpjson/internal/v2/value_tpl_test.go | 23 ++-- 9 files changed, 154 insertions(+), 77 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index 5ba339afdcd..92fefc55362 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -88,23 +88,21 @@ func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformCont } } -func (iter *pageIterator) next() (*transformable, int, bool, error) { +func (iter *pageIterator) next() (*response, bool, error) { if iter == nil || iter.resp == nil || iter.done { - return nil, 0, false, nil + return nil, false, nil } if iter.isFirst { - iter.pagination.log.Debug("first page requested") iter.isFirst = false tr, err := iter.getPage() if err != nil { - return nil, 0, false, err + return nil, false, err } if iter.pagination.requestFactory == nil { - iter.pagination.log.Debug("last page") iter.done = true } - return tr, iter.n, true, nil + return tr, true, nil } httpReq, err := iter.pagination.requestFactory.newHTTPRequest(iter.stdCtx, iter.trCtx) @@ -112,52 +110,52 @@ func (iter *pageIterator) next() (*transformable, int, bool, error) { if err == errNewURLValueNotSet { // if this error happens here it means the transform used to pick the new url.value // did not find any new value and we can stop paginating without error - iter.pagination.log.Debug("last page") iter.done = true - return nil, 0, false, nil + return nil, false, nil } - return nil, 0, false, err + return nil, false, err } resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) if err != nil { - return nil, 0, false, err + return nil, false, err } iter.resp = resp - tr, err := iter.getPage() + r, err := iter.getPage() if err != nil { - return nil, 0, false, err + return nil, false, err } - if len(tr.body) == 0 { + if r.body == nil { iter.pagination.log.Debug("finished pagination because there is no body") iter.done = true - return nil, 0, false, nil + return nil, false, nil } - return tr, iter.n, true, nil + return r, true, nil } -func (iter *pageIterator) getPage() (*transformable, error) { +func (iter *pageIterator) getPage() (*response, error) { bodyBytes, err := ioutil.ReadAll(iter.resp.Body) if err != nil { return nil, err } iter.resp.Body.Close() - tr := emptyTransformable() - tr.header = iter.resp.Header - tr.url = *iter.resp.Request.URL + var r response + r.header = iter.resp.Header + r.url = *iter.resp.Request.URL + r.page = iter.n if len(bodyBytes) > 0 { - if err := json.Unmarshal(bodyBytes, &tr.body); err != nil { + if err := json.Unmarshal(bodyBytes, &r.body); err != nil { return nil, err } } iter.n += 1 - return tr, nil + return &r, nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go index db8b8760a25..c2c96dc1aeb 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/stretchr/testify/assert" ) @@ -29,6 +30,7 @@ func TestGetRateLimitCase1(t *testing.T) { limit: tplLimit, reset: tplReset, remaining: tplRemaining, + log: logp.NewLogger(""), } resp := &http.Response{Header: header} epoch, err := rateLimit.getRateLimit(resp) @@ -52,6 +54,7 @@ func TestGetRateLimitCase2(t *testing.T) { limit: tplLimit, reset: tplReset, remaining: tplRemaining, + log: logp.NewLogger(""), } resp := &http.Response{Header: header} epoch, err := rateLimit.getRateLimit(resp) @@ -79,6 +82,7 @@ func TestGetRateLimitCase3(t *testing.T) { limit: tplLimit, reset: tplReset, remaining: tplRemaining, + log: logp.NewLogger(""), } resp := &http.Response{Header: header} epoch2, err := rateLimit.getRateLimit(resp) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/response.go b/x-pack/filebeat/input/httpjson/internal/v2/response.go index 9f98720376e..0e881e3697c 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/response.go @@ -7,7 +7,9 @@ package v2 import ( "context" "net/http" + "net/url" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -19,6 +21,13 @@ func registerResponseTransforms() { registerTransform(responseNamespace, setName, newSetResponse) } +type response struct { + page int + url url.URL + header http.Header + body interface{} +} + type responseProcessor struct { log *logp.Logger transforms []basicTransform @@ -52,48 +61,97 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans iter := rp.pagination.newPageIterator(stdCtx, trCtx, resp) for { - page, pageN, hasNext, err := iter.next() + page, hasNext, err := iter.next() if err != nil { ch <- maybeMsg{err: err} return } - if !hasNext || len(page.body) == 0 { + if !hasNext { + return + } + + respTrs := page.asTransformables(rp.log) + + if len(respTrs) == 0 { return } - *trCtx.lastPage = pageN - *trCtx.lastResponse = *page.clone() + *trCtx.lastResponse = *page rp.log.Debugf("last received page: %#v", trCtx.lastResponse) - for _, t := range rp.transforms { - page, err = t.run(trCtx, page) - if err != nil { - rp.log.Debug("error transforming page") - ch <- maybeMsg{err: err} - return + for _, tr := range respTrs { + for _, t := range rp.transforms { + tr, err = t.run(trCtx, tr) + if err != nil { + ch <- maybeMsg{err: err} + return + } } - } - - if rp.split == nil { - ch <- maybeMsg{msg: page.body} - rp.log.Debug("no split found: continuing to next page") - continue - } - if err := rp.split.run(trCtx, page, ch); err != nil { - if err == errEmptyField { - // nothing else to send for this page - rp.log.Debug("split operation finished") + if rp.split == nil { + ch <- maybeMsg{msg: tr.body} + rp.log.Debug("no split found: continuing") continue } - rp.log.Debug("split operation failed") - ch <- maybeMsg{err: err} - return + + if err := rp.split.run(trCtx, tr, ch); err != nil { + if err == errEmptyField { + // nothing else to send for this page + rp.log.Debug("split operation finished") + continue + } + rp.log.Debug("split operation failed") + ch <- maybeMsg{err: err} + return + } } } }() return ch, nil } + +func (resp *response) asTransformables(log *logp.Logger) []*transformable { + var ts []*transformable + + convertAndAppend := func(m map[string]interface{}) { + tr := emptyTransformable() + tr.header = resp.header.Clone() + tr.url = resp.url + tr.body = common.MapStr(m).Clone() + ts = append(ts, tr) + } + + switch tresp := resp.body.(type) { + case []interface{}: + for _, v := range tresp { + m, ok := v.(map[string]interface{}) + if !ok { + log.Debugf("events must be JSON objects, but got %T: skipping", v) + continue + } + convertAndAppend(m) + } + case map[string]interface{}: + convertAndAppend(tresp) + default: + log.Debugf("response is not a valid JSON") + } + + return ts +} + +func (resp *response) templateValues() common.MapStr { + if resp == nil { + return common.MapStr{} + } + return common.MapStr{ + "header": resp.header.Clone(), + "page": resp.page, + "url.value": resp.url.String(), + "params": resp.url.Query(), + "body": resp.body, + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go index 80a21bb4d40..eb72aa8de00 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split.go @@ -13,8 +13,8 @@ import ( ) var ( - errEmptyField = errors.New("the requested field is emtpy") - errEmptyTopField = errors.New("the requested top split field is emtpy") + errEmptyField = errors.New("the requested field is empty") + errEmptyTopField = errors.New("the requested top split field is empty") ) type split struct { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go index 4fb0eca1e20..c603134e248 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go @@ -246,7 +246,7 @@ func TestSplit(t *testing.T) { expectedErr: nil, }, { - name: "A nested array with an empty nested array in an object", + name: "A nested array with an empty nested array in an object published and returns error", config: &splitConfig{ Target: "body.response", Type: "array", @@ -276,7 +276,26 @@ func TestSplit(t *testing.T) { }, }, }, - expectedErr: nil, + expectedErr: errEmptyField, + }, + { + name: "First level split skips publish if no events and keep_parent: false", + config: &splitConfig{ + Target: "body.response", + Type: "array", + Split: &splitConfig{ + Target: "body.Event.Attributes", + KeepParent: false, + }, + }, + ctx: emptyTransformContext(), + resp: &transformable{ + body: common.MapStr{ + "response": []interface{}{}, + }, + }, + expectedMessages: []common.MapStr{}, + expectedErr: errEmptyTopField, }, } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform.go b/x-pack/filebeat/input/httpjson/internal/v2/transform.go index fa807728ccc..f4cc7a543ca 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform.go @@ -24,16 +24,14 @@ type transforms []transform type transformContext struct { cursor *cursor lastEvent *common.MapStr - lastPage *int - lastResponse *transformable + lastResponse *response } func emptyTransformContext() transformContext { return transformContext{ cursor: &cursor{}, lastEvent: &common.MapStr{}, - lastPage: new(int), - lastResponse: emptyTransformable(), + lastResponse: &response{}, } } @@ -64,6 +62,15 @@ func (t *transformable) clone() *transformable { } } +func (t *transformable) templateValues() common.MapStr { + return common.MapStr{ + "header": t.header.Clone(), + "body": t.body.Clone(), + "url.value": t.url.String(), + "url.params": t.url.Query(), + } +} + func emptyTransformable() *transformable { return &transformable{ body: common.MapStr{}, diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go index 4209f1e0ac2..f012c33736a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go @@ -18,7 +18,7 @@ func TestEmptyTransformContext(t *testing.T) { ctx := emptyTransformContext() assert.Equal(t, &cursor{}, ctx.cursor) assert.Equal(t, &common.MapStr{}, ctx.lastEvent) - assert.Equal(t, emptyTransformable(), ctx.lastResponse) + assert.Equal(t, &response{}, ctx.lastResponse) } func TestEmptyTransformable(t *testing.T) { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index 530f0061067..8cb8ad041c9 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -12,7 +12,6 @@ import ( "text/template" "time" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -65,19 +64,10 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal }() buf := new(bytes.Buffer) - data := common.MapStr{} - - _, _ = data.Put("header", tr.header.Clone()) - _, _ = data.Put("body", tr.body.Clone()) - _, _ = data.Put("url.value", tr.url.String()) - _, _ = data.Put("url.params", tr.url.Query()) + data := tr.templateValues() _, _ = data.Put("cursor", trCtx.cursor.clone()) _, _ = data.Put("last_event", trCtx.lastEvent.Clone()) - _, _ = data.Put("last_response.page", trCtx.lastPage) - _, _ = data.Put("last_response.body", trCtx.lastResponse.body.Clone()) - _, _ = data.Put("last_response.header", trCtx.lastResponse.header.Clone()) - _, _ = data.Put("last_response.url.value", trCtx.lastResponse.url.String()) - _, _ = data.Put("last_response.url.params", trCtx.lastResponse.url.Query()) + _, _ = data.Put("last_response", trCtx.lastResponse.templateValues()) if err := t.Template.Execute(buf, data); err != nil { return fallback(err) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 33dc1d5f0b5..6bb66fddf54 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -31,7 +31,7 @@ func TestValueTpl(t *testing.T) { value: "{{.last_response.body.param}}", paramCtx: transformContext{ lastEvent: &common.MapStr{}, - lastResponse: newTestTransformable(common.MapStr{"param": 25}, nil, ""), + lastResponse: newTestResponse(common.MapStr{"param": 25}, nil, ""), }, paramTr: emptyTransformable(), paramDefVal: "", @@ -41,8 +41,7 @@ func TestValueTpl(t *testing.T) { name: "can render default value if execute fails", value: "{{.last_response.body.does_not_exist}}", paramCtx: transformContext{ - lastEvent: &common.MapStr{}, - lastResponse: emptyTransformable(), + lastEvent: &common.MapStr{}, }, paramTr: emptyTransformable(), paramDefVal: "25", @@ -161,7 +160,7 @@ func TestValueTpl(t *testing.T) { value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, paramCtx: transformContext{ lastEvent: &common.MapStr{}, - lastResponse: newTestTransformable( + lastResponse: newTestResponse( nil, http.Header{"Link": []string{ `; title="Page 3"; rel="next"`, @@ -177,7 +176,7 @@ func TestValueTpl(t *testing.T) { name: "func getRFC5988Link does not match", value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, paramCtx: transformContext{ - lastResponse: newTestTransformable( + lastResponse: newTestResponse( nil, http.Header{"Link": []string{ ``, @@ -227,16 +226,18 @@ func TestValueTpl(t *testing.T) { } } -func newTestTransformable(body common.MapStr, header http.Header, url string) *transformable { - tr := emptyTransformable() +func newTestResponse(body common.MapStr, header http.Header, url string) *response { + resp := &response{ + header: http.Header{}, + } if len(body) > 0 { - tr.body = body + resp.body = body } if len(header) > 0 { - tr.header = header + resp.header = header } if url != "" { - tr.url = newURL(url) + resp.url = newURL(url) } - return tr + return resp } From 3a427c99cb2b89ded185308900c77acc24c86860 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 30 Nov 2020 16:37:00 +0100 Subject: [PATCH 31/35] Add tests cases and minor changes --- .../httpjson/internal/v2/config_oauth_test.go | 81 --- .../input/httpjson/internal/v2/config_test.go | 372 +++++++++++++ .../input/httpjson/internal/v2/cursor.go | 4 +- .../input/httpjson/internal/v2/input_test.go | 519 ++++++++++++++++++ .../input/httpjson/internal/v2/pagination.go | 4 +- .../httpjson/internal/v2/rate_limiter.go | 6 +- .../httpjson/internal/v2/rate_limiter_test.go | 11 +- .../input/httpjson/internal/v2/request.go | 43 +- .../input/httpjson/internal/v2/response.go | 39 +- .../input/httpjson/internal/v2/split.go | 131 ++--- .../input/httpjson/internal/v2/split_test.go | 83 ++- .../internal/v2/testdata/credentials.json | 7 + .../v2/testdata/invalid_credentials.json | 1 + .../input/httpjson/internal/v2/transform.go | 142 +++-- .../httpjson/internal/v2/transform_append.go | 28 +- .../internal/v2/transform_append_test.go | 26 +- .../httpjson/internal/v2/transform_delete.go | 26 +- .../internal/v2/transform_delete_test.go | 18 +- .../httpjson/internal/v2/transform_set.go | 32 +- .../internal/v2/transform_set_test.go | 32 +- .../httpjson/internal/v2/transform_test.go | 26 +- .../input/httpjson/internal/v2/value_tpl.go | 12 +- .../httpjson/internal/v2/value_tpl_test.go | 50 +- 23 files changed, 1323 insertions(+), 370 deletions(-) delete mode 100644 x-pack/filebeat/input/httpjson/internal/v2/config_oauth_test.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/config_test.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/input_test.go create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json create mode 100644 x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_oauth_test.go b/x-pack/filebeat/input/httpjson/internal/v2/config_oauth_test.go deleted file mode 100644 index d495bec7283..00000000000 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_oauth_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package v2 - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestProviderCanonical(t *testing.T) { - const ( - a oAuth2Provider = "gOoGle" - b oAuth2Provider = "google" - ) - - assert.Equal(t, a.canonical(), b.canonical()) -} - -func TestGetProviderIsCanonical(t *testing.T) { - const expected oAuth2Provider = "google" - - oauth2 := oAuth2Config{Provider: "GOogle"} - assert.Equal(t, expected, oauth2.getProvider()) -} - -func TestIsEnabled(t *testing.T) { - oauth2 := oAuth2Config{} - if !oauth2.isEnabled() { - t.Fatal("OAuth2 should be enabled by default") - } - - var enabled = false - oauth2.Enabled = &enabled - - assert.False(t, oauth2.isEnabled()) - - enabled = true - - assert.True(t, oauth2.isEnabled()) -} - -func TestGetTokenURL(t *testing.T) { - const expected = "http://localhost" - oauth2 := oAuth2Config{TokenURL: "http://localhost"} - assert.Equal(t, expected, oauth2.getTokenURL()) -} - -func TestGetTokenURLWithAzure(t *testing.T) { - const expectedWithoutTenantID = "http://localhost" - oauth2 := oAuth2Config{TokenURL: "http://localhost", Provider: "azure"} - - assert.Equal(t, expectedWithoutTenantID, oauth2.getTokenURL()) - - oauth2.TokenURL = "" - oauth2.AzureTenantID = "a_tenant_id" - const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" - - assert.Equal(t, expectedWithTenantID, oauth2.getTokenURL()) - -} - -func TestGetEndpointParams(t *testing.T) { - var expected = map[string][]string{"foo": {"bar"}} - oauth2 := oAuth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} - assert.Equal(t, expected, oauth2.getEndpointParams()) -} - -func TestGetEndpointParamsWithAzure(t *testing.T) { - var expectedWithoutResource = map[string][]string{"foo": {"bar"}} - oauth2 := oAuth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} - - assert.Equal(t, expectedWithoutResource, oauth2.getEndpointParams()) - - oauth2.AzureResource = "baz" - var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} - - assert.Equal(t, expectedWithResource, oauth2.getEndpointParams()) -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go new file mode 100644 index 00000000000..19693f0e727 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go @@ -0,0 +1,372 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/oauth2/google" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestProviderCanonical(t *testing.T) { + const ( + a oAuth2Provider = "gOoGle" + b oAuth2Provider = "google" + ) + + assert.Equal(t, a.canonical(), b.canonical()) +} + +func TestGetProviderIsCanonical(t *testing.T) { + const expected oAuth2Provider = "google" + + oauth2 := oAuth2Config{Provider: "GOogle"} + assert.Equal(t, expected, oauth2.getProvider()) +} + +func TestIsEnabled(t *testing.T) { + oauth2 := oAuth2Config{} + if !oauth2.isEnabled() { + t.Fatal("OAuth2 should be enabled by default") + } + + var enabled = false + oauth2.Enabled = &enabled + + assert.False(t, oauth2.isEnabled()) + + enabled = true + + assert.True(t, oauth2.isEnabled()) +} + +func TestGetTokenURL(t *testing.T) { + const expected = "http://localhost" + oauth2 := oAuth2Config{TokenURL: "http://localhost"} + assert.Equal(t, expected, oauth2.getTokenURL()) +} + +func TestGetTokenURLWithAzure(t *testing.T) { + const expectedWithoutTenantID = "http://localhost" + oauth2 := oAuth2Config{TokenURL: "http://localhost", Provider: "azure"} + + assert.Equal(t, expectedWithoutTenantID, oauth2.getTokenURL()) + + oauth2.TokenURL = "" + oauth2.AzureTenantID = "a_tenant_id" + const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" + + assert.Equal(t, expectedWithTenantID, oauth2.getTokenURL()) + +} + +func TestGetEndpointParams(t *testing.T) { + var expected = map[string][]string{"foo": {"bar"}} + oauth2 := oAuth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} + assert.Equal(t, expected, oauth2.getEndpointParams()) +} + +func TestGetEndpointParamsWithAzure(t *testing.T) { + var expectedWithoutResource = map[string][]string{"foo": {"bar"}} + oauth2 := oAuth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} + + assert.Equal(t, expectedWithoutResource, oauth2.getEndpointParams()) + + oauth2.AzureResource = "baz" + var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} + + assert.Equal(t, expectedWithResource, oauth2.getEndpointParams()) +} + +func TestConfigFailsWithInvalidMethod(t *testing.T) { + m := map[string]interface{}{ + "request.method": "DELETE", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + if err := cfg.Unpack(&conf); err == nil { + t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") + } +} + +func TestConfigMustFailWithInvalidURL(t *testing.T) { + m := map[string]interface{}{ + "request.url": "::invalid::", + } + cfg := common.MustNewConfigFrom(m) + conf := defaultConfig() + err := cfg.Unpack(&conf) + assert.EqualError(t, err, `parse "::invalid::": missing protocol scheme accessing 'request.url'`) +} + +func TestConfigOauth2Validation(t *testing.T) { + cases := []struct { + name string + expectedErr string + input map[string]interface{} + setup func() + teardown func() + }{ + { + name: "can't set oauth2 and basic auth together", + expectedErr: "only one kind of auth can be enabled accessing 'auth'", + input: map[string]interface{}{ + "auth.basic.user": "user", + "auth.basic.password": "pass", + "auth.oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + }, + }, + { + name: "can set oauth2 and basic auth together if oauth2 is disabled", + input: map[string]interface{}{ + "auth.basic.user": "user", + "auth.basic.password": "pass", + "auth.oauth2": map[string]interface{}{ + "enabled": false, + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + }, + }, + { + name: "token_url and client credentials must be set", + expectedErr: "both token_url and client credentials must be provided accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{}, + }, + }, + { + name: "must fail with an unknown provider", + expectedErr: "unknown provider \"unknown\" accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "unknown", + }, + }, + }, + { + name: "azure must have either tenant_id or token_url", + expectedErr: "at least one of token_url or tenant_id must be provided accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "azure", + }, + }, + }, + { + name: "azure must have only one of token_url and tenant_id", + expectedErr: "only one of token_url and tenant_id can be used accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + "token_url": "localhost", + }, + }, + }, + { + name: "azure must have client credentials set", + expectedErr: "client credentials must be provided accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "azure", + "azure.tenant_id": "a_tenant_id", + }, + }, + }, + { + name: "azure config is valid", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "azure", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + }, + }, + }, + { + name: "google can't have token_url or client credentials set", + expectedErr: "none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "azure": map[string]interface{}{ + "tenant_id": "a_tenant_id", + }, + "client.id": "a_client_id", + "client.secret": "a_client_secret", + "token_url": "localhost", + }, + }, + }, + { + name: "google must fail if no ADC available", + expectedErr: "no authentication credentials were configured or detected (ADC) accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + }, + }, + setup: func() { + // we change the default function to force a failure + findDefaultGoogleCredentials = func(context.Context, ...string) (*google.Credentials, error) { + return nil, errors.New("failed") + } + }, + teardown: func() { findDefaultGoogleCredentials = google.FindDefaultCredentials }, + }, + { + name: "google must fail if credentials file not found", + expectedErr: "the file \"./wrong\" cannot be found accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./wrong", + }, + }, + }, + { + name: "google must fail if ADC is wrongly set", + expectedErr: "no authentication credentials were configured or detected (ADC) accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + }, + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./wrong") }, + }, + { + name: "google must work if ADC is set up", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + }, + }, + setup: func() { os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "./testdata/credentials.json") }, + }, + { + name: "google must work if credentials_file is correct", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + }, + }, + }, + { + name: "google must work if jwt_file is correct", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + }, + }, + }, + { + name: "google must work if credentials_json is correct", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" + }`), + }, + }, + }, + { + name: "google must fail if credentials_json is not a valid JSON", + expectedErr: "google.credentials_json must be valid JSON accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_json": []byte(`invalid`), + }, + }, + }, + { + name: "google must fail if the provided credentials file is not a valid JSON", + expectedErr: "the file \"./testdata/invalid_credentials.json\" does not contain valid JSON accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/invalid_credentials.json", + }, + }, + }, + { + name: "google must fail if the delegated_account is set without jwt_file", + expectedErr: "google.delegated_account can only be provided with a jwt_file accessing 'auth.oauth2'", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + "google.delegated_account": "delegated@account.com", + }, + }, + }, + { + name: "google must work with delegated_account and a valid jwt_file", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + "google.delegated_account": "delegated@account.com", + }, + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + if c.setup != nil { + c.setup() + } + + if c.teardown != nil { + defer c.teardown() + } + + c.input["request.url"] = "localhost" + cfg := common.MustNewConfigFrom(c.input) + conf := defaultConfig() + err := cfg.Unpack(&conf) + + switch { + case c.expectedErr == "": + if err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } + + case c.expectedErr != "": + if err == nil || err.Error() != c.expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) + } + } + }) + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go b/x-pack/filebeat/input/httpjson/internal/v2/cursor.go index bb02daecf50..053cdd87bd4 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/cursor.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/cursor.go @@ -40,7 +40,7 @@ func (c *cursor) load(cursor *inputcursor.Cursor) { c.log.Debugf("cursor loaded: %v", c.state) } -func (c *cursor) update(trCtx transformContext) { +func (c *cursor) update(trCtx *transformContext) { if c.cfg == nil { return } @@ -50,7 +50,7 @@ func (c *cursor) update(trCtx transformContext) { } for k, cfg := range c.cfg { - v := cfg.Value.Execute(trCtx, emptyTransformable(), cfg.Default, c.log) + v := cfg.Value.Execute(trCtx, transformable{}, cfg.Default, c.log) _, _ = c.state.Put(k, v) c.log.Debugf("cursor.%s stored with %s", k, v) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_test.go b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go new file mode 100644 index 00000000000..3f9fc342c1f --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_test.go @@ -0,0 +1,519 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "context" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "golang.org/x/sync/errgroup" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + beattest "github.com/elastic/beats/v7/libbeat/publisher/testing" +) + +func TestInput(t *testing.T) { + testCases := []struct { + name string + setupServer func(*testing.T, http.HandlerFunc, map[string]interface{}) + baseConfig map[string]interface{} + handler http.HandlerFunc + expected []string + }{ + { + name: "Test simple GET request", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test simple HTTPS GET request", + setupServer: newTestServer(httptest.NewTLSServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "request.ssl.verification_mode": "none", + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test request honors rate limit", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "http_method": "GET", + "request.rate_limit.limit": `{{.last_request.header.Get "X-Rate-Limit-Limit"}}`, + "request.rate_limit.remaining": `{{.last_request.header.Get "X-Rate-Limit-Remaining"}}`, + "request.rate_limit.reset": `{{.last_request.header.Get "X-Rate-Limit-Reset"}}`, + }, + handler: rateLimitHandler(), + expected: []string{`{"hello":"world"}`}, + }, + { + name: "Test request retries when failed", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + }, + handler: retryHandler(), + expected: []string{`{"hello":"world"}`}, + }, + { + name: "Test POST request with body", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "request.body": map[string]interface{}{ + "test": "abc", + }, + }, + handler: defaultHandler("POST", `{"test":"abc"}`), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test repeated POST requests", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": "100ms", + "request.method": "POST", + }, + handler: defaultHandler("POST", ""), + expected: []string{ + `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`, + `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`, + }, + }, + { + name: "Test split by json objects array", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.hello", + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"world":"moon"}`, `{"space":[{"cake":"pumpkin"}]}`}, + }, + { + name: "Test split by json objects array with keep parent", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.hello", + "keep_parent": true, + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{ + `{"hello":{"world":"moon"}}`, + `{"hello":{"space":[{"cake":"pumpkin"}]}}`, + }, + }, + { + name: "Test nested split", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.hello", + "split": map[string]interface{}{ + "target": "body.space", + "keep_parent": true, + }, + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{ + `{"world":"moon"}`, + `{"space":{"cake":"pumpkin"}}`, + }, + }, + { + name: "Test split events by not found", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.unknown", + }, + }, + handler: defaultHandler("GET", ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, + { + name: "Test date cursor", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerRequestTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + // mock timeNow func to return a fixed value + timeNow = func() time.Time { + t, _ := time.Parse(time.RFC3339, "2002-10-02T15:00:00Z") + return t + } + + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + t.Cleanup(func() { timeNow = time.Now }) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "request.transforms": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.$filter", + "value": "alertCreationTime ge {{.cursor.timestamp}}", + "default": `alertCreationTime ge {{formatDate (now (parseDuration "-10m")) "2006-01-02T15:04:05Z"}}`, + }, + }, + }, + "cursor": map[string]interface{}{ + "timestamp": map[string]interface{}{ + "value": `{{index .last_response.body "@timestamp"}}`, + }, + }, + }, + handler: dateCursorHandler(), + expected: []string{ + `{"@timestamp":"2002-10-02T15:00:00Z","foo":"bar"}`, + `{"@timestamp":"2002-10-02T15:00:01Z","foo":"bar"}`, + `{"@timestamp":"2002-10-02T15:00:02Z","foo":"bar"}`, + }, + }, + { + name: "Test pagination", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerPaginationTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.split": map[string]interface{}{ + "target": "body.items", + }, + "response.pagination": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.page", + "value": "{{.last_response.body.nextPageToken}}", + }, + }, + }, + }, + handler: paginationHandler(), + expected: []string{`{"foo":"bar"}`, `{"foo":"bar"}`}, + }, + { + name: "Test pagination with array response", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerPaginationTransforms() + t.Cleanup(func() { registeredTransforms = newRegistry() }) + server := httptest.NewServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "GET", + "response.pagination": []interface{}{ + map[string]interface{}{ + "set": map[string]interface{}{ + "target": "url.params.page", + "value": `{{index (index .last_response.body 0) "nextPageToken"}}`, + }, + }, + }, + }, + handler: paginationArrayHandler(), + expected: []string{`{"nextPageToken":"bar","foo":"bar"}`, `{"foo":"bar"}`, `{"foo":"bar"}`}, + }, + { + name: "Test oauth2", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + server := httptest.NewServer(h) + config["request.url"] = server.URL + config["auth.oauth2.token_url"] = server.URL + "/token" + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": "POST", + "auth.oauth2.client.id": "a_client_id", + "auth.oauth2.client.secret": "a_client_secret", + "auth.oauth2.endpoint_params": map[string]interface{}{ + "param1": "v1", + }, + "auth.oauth2.scopes": []string{"scope1", "scope2"}, + }, + handler: oauth2Handler, + expected: []string{`{"hello": "world"}`}, + }, + } + + for _, testCase := range testCases { + tc := testCase + t.Run(tc.name, func(t *testing.T) { + tc.setupServer(t, tc.handler, tc.baseConfig) + + cfg := common.MustNewConfigFrom(tc.baseConfig) + + conf := defaultConfig() + assert.NoError(t, cfg.Unpack(&conf)) + + input, err := newStatelessInput(conf) + + assert.NoError(t, err) + assert.Equal(t, "httpjson-stateless", input.Name()) + assert.NoError(t, input.Test(v2.TestContext{})) + + chanClient := beattest.NewChanClient(len(tc.expected)) + t.Cleanup(func() { _ = chanClient.Close() }) + + ctx, cancel := newV2Context() + t.Cleanup(cancel) + + var g errgroup.Group + g.Go(func() error { + return input.Run(ctx, chanClient) + }) + + timeout := time.NewTimer(5 * time.Second) + t.Cleanup(func() { _ = timeout.Stop() }) + + var receivedCount int + wait: + for { + select { + case <-timeout.C: + t.Errorf("timed out waiting for %d events", len(tc.expected)) + cancel() + return + case got := <-chanClient.Channel: + val, err := got.Fields.GetValue("message") + assert.NoError(t, err) + assert.JSONEq(t, tc.expected[receivedCount], val.(string)) + receivedCount += 1 + if receivedCount == len(tc.expected) { + cancel() + break wait + } + } + } + assert.NoError(t, g.Wait()) + }) + } +} + +func newTestServer( + newServer func(http.Handler) *httptest.Server, +) func(*testing.T, http.HandlerFunc, map[string]interface{}) { + return func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + server := newServer(h) + config["request.url"] = server.URL + t.Cleanup(server.Close) + } +} + +func newV2Context() (v2.Context, func()) { + ctx, cancel := context.WithCancel(context.Background()) + return v2.Context{ + Logger: logp.NewLogger("httpjson_test"), + ID: "test_id", + Cancelation: ctx, + }, cancel +} + +func defaultHandler(expectedMethod, expectedBody string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + msg := `{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}` + switch { + case r.Method != expectedMethod: + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected method was %q"}`, expectedMethod) + case expectedBody != "": + body, _ := ioutil.ReadAll(r.Body) + r.Body.Close() + if expectedBody != string(body) { + w.WriteHeader(http.StatusBadRequest) + msg = fmt.Sprintf(`{"error":"expected body was %q"}`, expectedBody) + } + } + + _, _ = w.Write([]byte(msg)) + } +} + +func rateLimitHandler() http.HandlerFunc { + var isRetry bool + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if isRetry { + _, _ = w.Write([]byte(`{"hello":"world"}`)) + return + } + w.Header().Set("X-Rate-Limit-Limit", "0") + w.Header().Set("X-Rate-Limit-Remaining", "0") + w.Header().Set("X-Rate-Limit-Reset", fmt.Sprint(time.Now().Unix())) + w.WriteHeader(http.StatusTooManyRequests) + isRetry = true + _, _ = w.Write([]byte(`{"error":"too many requests"}`)) + } +} + +func retryHandler() http.HandlerFunc { + count := 0 + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if count == 2 { + _, _ = w.Write([]byte(`{"hello":"world"}`)) + return + } + w.WriteHeader(rand.Intn(100) + 500) + count += 1 + } +} + +func oauth2TokenHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + _ = r.ParseForm() + switch { + case r.Method != "POST": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + case r.FormValue("grant_type") != "client_credentials": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong grant_type"}`)) + case r.FormValue("client_id") != "a_client_id": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong client_id"}`)) + case r.FormValue("client_secret") != "a_client_secret": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong client_secret"}`)) + case r.FormValue("scope") != "scope1 scope2": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong scope"}`)) + case r.FormValue("param1") != "v1": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong param1"}`)) + default: + _, _ = w.Write([]byte(`{"token_type": "Bearer", "expires_in": "60", "access_token": "abcd"}`)) + } +} + +func oauth2Handler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/token" { + oauth2TokenHandler(w, r) + return + } + + w.Header().Set("content-type", "application/json") + switch { + case r.Method != "POST": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong method"}`)) + case r.Header.Get("Authorization") != "Bearer abcd": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong bearer"}`)) + default: + _, _ = w.Write([]byte(`{"hello":"world"}`)) + } +} + +func dateCursorHandler() http.HandlerFunc { + var count int + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + switch count { + case 0: + if r.URL.Query().Get("$filter") != "alertCreationTime ge 2002-10-02T14:50:00Z" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong initial cursor value"`)) + return + } + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:00Z","foo":"bar"}`)) + case 1: + if r.URL.Query().Get("$filter") != "alertCreationTime ge 2002-10-02T15:00:00Z" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong cursor value"`)) + return + } + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:01Z","foo":"bar"}`)) + case 2: + if r.URL.Query().Get("$filter") != "alertCreationTime ge 2002-10-02T15:00:01Z" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong cursor value"`)) + return + } + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:02Z","foo":"bar"}`)) + } + count += 1 + } +} + +func paginationHandler() http.HandlerFunc { + var count int + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + switch count { + case 0: + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:00Z","nextPageToken":"bar","items":[{"foo":"bar"}]}`)) + case 1: + if r.URL.Query().Get("page") != "bar" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong page token value"}`)) + return + } + _, _ = w.Write([]byte(`{"@timestamp":"2002-10-02T15:00:01Z","items":[{"foo":"bar"}]}`)) + } + count += 1 + } +} + +func paginationArrayHandler() http.HandlerFunc { + var count int + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + switch count { + case 0: + _, _ = w.Write([]byte(`[{"nextPageToken":"bar","foo":"bar"},{"foo":"bar"}]`)) + case 1: + if r.URL.Query().Get("page") != "bar" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"wrong page token value"}`)) + return + } + _, _ = w.Write([]byte(`[{"foo":"bar"}]`)) + } + count += 1 + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index 92fefc55362..6a8a15105fd 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -68,7 +68,7 @@ type pageIterator struct { pagination *pagination stdCtx context.Context - trCtx transformContext + trCtx *transformContext resp *http.Response @@ -78,7 +78,7 @@ type pageIterator struct { n int } -func (p *pagination) newPageIterator(stdCtx context.Context, trCtx transformContext, resp *http.Response) *pageIterator { +func (p *pagination) newPageIterator(stdCtx context.Context, trCtx *transformContext, resp *http.Response) *pageIterator { return &pageIterator{ pagination: p, stdCtx: stdCtx, diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go index 29926ffff2a..5c7e2c16a98 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter.go @@ -101,8 +101,8 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { return 0, nil } - tr := emptyTransformable() - tr.header = resp.Header + tr := transformable{} + tr.setHeader(resp.Header) remaining := r.remaining.Execute(emptyTransformContext(), tr, nil, r.log) if remaining == "" { @@ -132,7 +132,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { return 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) } - if timeNow().Sub(time.Unix(epoch, 0)) > 0 { + if timeNow().Unix() > epoch { return 0, nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go index c2c96dc1aeb..cdaa4398d8a 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/rate_limiter_test.go @@ -10,12 +10,13 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/logp" ) // Test getRateLimit function with a remaining quota, expect to receive 0, nil. -func TestGetRateLimitCase1(t *testing.T) { +func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "118") @@ -38,8 +39,7 @@ func TestGetRateLimitCase1(t *testing.T) { assert.EqualValues(t, 0, epoch) } -// Test getRateLimit function with a past time, expect to receive 0, nil. -func TestGetRateLimitCase2(t *testing.T) { +func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { header := make(http.Header) header.Add("X-Rate-Limit-Limit", "10") header.Add("X-Rate-Limit-Remaining", "0") @@ -62,8 +62,7 @@ func TestGetRateLimitCase2(t *testing.T) { assert.EqualValues(t, 0, epoch) } -// Test getRateLimit function with a time yet to come, expect to receive , nil. -func TestGetRateLimitCase3(t *testing.T) { +func TestGetRateLimitReturnsResetValue(t *testing.T) { epoch := int64(1604582732 + 100) timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index a52964244df..f0eab710857 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -31,7 +31,7 @@ type httpClient struct { limiter *rateLimiter } -func (c *httpClient) do(stdCtx context.Context, trCtx transformContext, req *http.Request) (*http.Response, error) { +func (c *httpClient) do(stdCtx context.Context, trCtx *transformContext, req *http.Request) (*http.Response, error) { resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { return c.client.Do(req) }) @@ -46,25 +46,27 @@ func (c *httpClient) do(stdCtx context.Context, trCtx transformContext, req *htt return resp, nil } -func (rf *requestFactory) newRequest(ctx transformContext) (*transformable, error) { - req := emptyTransformable() - req.url = rf.url +func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, error) { + req := transformable{} + req.setURL(rf.url) - if rf.body != nil { - req.body.DeepUpdate(*rf.body) + if rf.body != nil && len(*rf.body) > 0 { + req.setBody(rf.body.Clone()) } - req.header.Set("Accept", "application/json") - req.header.Set("User-Agent", userAgent) + header := http.Header{} + header.Set("Accept", "application/json") + header.Set("User-Agent", userAgent) if rf.method == "POST" { - req.header.Set("Content-Type", "application/json") + header.Set("Content-Type", "application/json") } + req.setHeader(header) var err error for _, t := range rf.transforms { req, err = t.run(ctx, req) if err != nil { - return nil, err + return transformable{}, err } } @@ -100,17 +102,17 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. return rf } -func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transformContext) (*http.Request, error) { +func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx *transformContext) (*http.Request, error) { trReq, err := rf.newRequest(trCtx) if err != nil { return nil, err } var body []byte - if len(trReq.body) > 0 { + if len(trReq.body()) > 0 { switch rf.method { case "POST": - body, err = json.Marshal(trReq.body) + body, err = json.Marshal(trReq.body()) if err != nil { return nil, err } @@ -119,14 +121,15 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx transform } } - req, err := http.NewRequest(rf.method, trReq.url.String(), bytes.NewBuffer(body)) + url := trReq.url() + req, err := http.NewRequest(rf.method, url.String(), bytes.NewBuffer(body)) if err != nil { return nil, err } req = req.WithContext(stdCtx) - req.Header = trReq.header.Clone() + req.Header = trReq.header().Clone() if rf.user != "" || rf.password != "" { req.SetBasicAuth(rf.user, rf.password) @@ -155,7 +158,7 @@ func newRequester( } } -func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, publisher inputcursor.Publisher) error { +func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, publisher inputcursor.Publisher) error { req, err := r.requestFactory.newHTTPRequest(stdCtx, trCtx) if err != nil { return fmt.Errorf("failed to create http request: %w", err) @@ -185,16 +188,18 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx transformContext, pu continue } - if err := publisher.Publish(event, trCtx.cursor.clone()); err != nil { + if err := publisher.Publish(event, trCtx.cursorMap()); err != nil { r.log.Errorf("error publishing event: %v", err) continue } - *trCtx.lastEvent = maybeMsg.msg + trCtx.updateLastEvent(maybeMsg.msg) n += 1 } - trCtx.cursor.update(trCtx) + + trCtx.updateCursor() r.log.Infof("request finished: %d events published", n) + return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/response.go b/x-pack/filebeat/input/httpjson/internal/v2/response.go index 0e881e3697c..cc5f6605bc5 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/response.go @@ -28,6 +28,27 @@ type response struct { body interface{} } +func (resp *response) clone() *response { + clone := &response{ + page: resp.page, + header: resp.header.Clone(), + url: resp.url, + } + + switch t := resp.body.(type) { + case []interface{}: + c := make([]interface{}, len(t)) + copy(c, t) + clone.body = c + case common.MapStr: + clone.body = t.Clone() + case map[string]interface{}: + clone.body = common.MapStr(t).Clone() + } + + return clone +} + type responseProcessor struct { log *logp.Logger transforms []basicTransform @@ -53,7 +74,7 @@ func newResponseProcessor(config *responseConfig, pagination *pagination, log *l return rp } -func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx transformContext, resp *http.Response) (<-chan maybeMsg, error) { +func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx *transformContext, resp *http.Response) (<-chan maybeMsg, error) { ch := make(chan maybeMsg) go func() { @@ -77,7 +98,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans return } - *trCtx.lastResponse = *page + trCtx.updateLastResponse(*page) rp.log.Debugf("last received page: %#v", trCtx.lastResponse) @@ -91,7 +112,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans } if rp.split == nil { - ch <- maybeMsg{msg: tr.body} + ch <- maybeMsg{msg: tr.body()} rp.log.Debug("no split found: continuing") continue } @@ -113,14 +134,14 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx trans return ch, nil } -func (resp *response) asTransformables(log *logp.Logger) []*transformable { - var ts []*transformable +func (resp *response) asTransformables(log *logp.Logger) []transformable { + var ts []transformable convertAndAppend := func(m map[string]interface{}) { - tr := emptyTransformable() - tr.header = resp.header.Clone() - tr.url = resp.url - tr.body = common.MapStr(m).Clone() + tr := transformable{} + tr.setHeader(resp.header.Clone()) + tr.setURL(resp.url) + tr.setBody(common.MapStr(m).Clone()) ts = append(ts, tr) } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split.go b/x-pack/filebeat/input/httpjson/internal/v2/split.go index eb72aa8de00..17d5c2a7c9b 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split.go @@ -13,8 +13,9 @@ import ( ) var ( - errEmptyField = errors.New("the requested field is empty") - errEmptyTopField = errors.New("the requested top split field is empty") + errEmptyField = errors.New("the requested field is empty") + errExpectedSplitArr = errors.New("split was expecting field to be an array") + errExpectedSplitObj = errors.New("split was expecting field to be an object") ) type split struct { @@ -22,7 +23,7 @@ type split struct { targetInfo targetInfo kind string transforms []basicTransform - split *split + child *split keepParent bool keyField string } @@ -70,127 +71,115 @@ func newSplit(c *splitConfig, log *logp.Logger) (*split, error) { keepParent: c.KeepParent, keyField: c.KeyField, transforms: ts, - split: s, + child: s, }, nil } -func (s *split) run(ctx transformContext, resp *transformable, ch chan<- maybeMsg) error { - return s.runChild(ctx, resp, ch, false) +func (s *split) run(ctx *transformContext, resp transformable, ch chan<- maybeMsg) error { + root := resp.body() + return s.split(ctx, root, ch) } -func (s *split) runChild(ctx transformContext, resp *transformable, ch chan<- maybeMsg, isChild bool) error { - respCpy := resp.clone() - - v, err := respCpy.body.GetValue(s.targetInfo.Name) +func (s *split) split(ctx *transformContext, root common.MapStr, ch chan<- maybeMsg) error { + v, err := root.GetValue(s.targetInfo.Name) if err != nil && err != common.ErrKeyNotFound { return err } + if v == nil { + ch <- maybeMsg{msg: root} + return errEmptyField + } + switch s.kind { case "", splitTypeArr: - if v == nil { - s.log.Debug("array field is nil, sending main body") - return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) - } - - arr, ok := v.([]interface{}) + varr, ok := v.([]interface{}) if !ok { - return fmt.Errorf("field %s needs to be an array to be able to split on it but it is %T", s.targetInfo.Name, v) + return errExpectedSplitArr } - if len(arr) == 0 { - s.log.Debug("array field is empty, sending main body") - return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) + if len(varr) == 0 { + ch <- maybeMsg{msg: root} + return errEmptyField } - for _, a := range arr { - if err := s.sendEvent(ctx, respCpy, "", a, ch, isChild); err != nil { - return err + for _, e := range varr { + if err := s.sendMessage(ctx, root, "", e, ch); err != nil { + s.log.Debug(err) } } return nil case splitTypeMap: - if v == nil { - s.log.Debug("object field is nil, sending main body") - return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) - } - - ms, ok := toMapStr(v) + vmap, ok := toMapStr(v) if !ok { - return fmt.Errorf("field %s needs to be a map to be able to split on it but it is %T", s.targetInfo.Name, v) + return errExpectedSplitObj } - if len(ms) == 0 { - s.log.Debug("object field is empty, sending main body") - return s.sendEvent(ctx, respCpy, "", nil, ch, isChild) + if len(vmap) == 0 { + ch <- maybeMsg{msg: root} + return errEmptyField } - for k, v := range ms { - if err := s.sendEvent(ctx, respCpy, k, v, ch, isChild); err != nil { - return err + for k, e := range vmap { + if err := s.sendMessage(ctx, root, k, e, ch); err != nil { + s.log.Debug(err) } } return nil } - return errors.New("invalid split type") -} - -func toMapStr(v interface{}) (common.MapStr, bool) { - var m common.MapStr - if v == nil { - return m, true - } - switch ts := v.(type) { - case common.MapStr: - m = ts - case map[string]interface{}: - m = common.MapStr(ts) - default: - return nil, false - } - return m, true + return errors.New("unknown split type") } -func (s *split) sendEvent(ctx transformContext, resp *transformable, key string, val interface{}, ch chan<- maybeMsg, isChild bool) error { - if val == nil && !isChild && !s.keepParent { - return errEmptyTopField - } - - m, ok := toMapStr(val) +func (s *split) sendMessage(ctx *transformContext, root common.MapStr, key string, v interface{}, ch chan<- maybeMsg) error { + obj, ok := toMapStr(v) if !ok { - return errors.New("split can only be applied on object lists") + return errExpectedSplitObj } + clone := root.Clone() + if s.keyField != "" && key != "" { - _, _ = m.Put(s.keyField, key) + _, _ = obj.Put(s.keyField, key) } if s.keepParent { - _, _ = resp.body.Put(s.targetInfo.Name, m) + _, _ = clone.Put(s.targetInfo.Name, obj) } else { - resp.body = m + clone = obj } + tr := transformable{} + tr.setBody(clone) + var err error for _, t := range s.transforms { - resp, err = t.run(ctx, resp) + tr, err = t.run(ctx, tr) if err != nil { return err } } - if s.split != nil { - return s.split.runChild(ctx, resp, ch, true) + if s.child != nil { + return s.child.split(ctx, clone, ch) } - ch <- maybeMsg{msg: resp.body.Clone()} - - if val == nil { - return errEmptyField - } + ch <- maybeMsg{msg: clone} return nil } + +func toMapStr(v interface{}) (common.MapStr, bool) { + if v == nil { + return common.MapStr{}, false + } + switch t := v.(type) { + case common.MapStr: + return t, true + case map[string]interface{}: + return common.MapStr(t), true + } + return common.MapStr{}, false +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go index c603134e248..2e1f7c59b70 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/split_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/split_test.go @@ -19,8 +19,8 @@ func TestSplit(t *testing.T) { cases := []struct { name string config *splitConfig - ctx transformContext - resp *transformable + ctx *transformContext + resp transformable expectedMessages []common.MapStr expectedErr error }{ @@ -37,8 +37,8 @@ func TestSplit(t *testing.T) { }, }, ctx: emptyTransformContext(), - resp: &transformable{ - body: common.MapStr{ + resp: transformable{ + "body": common.MapStr{ "this": "is kept", "alerts": []interface{}{ map[string]interface{}{ @@ -104,8 +104,8 @@ func TestSplit(t *testing.T) { }, }, ctx: emptyTransformContext(), - resp: &transformable{ - body: common.MapStr{ + resp: transformable{ + "body": common.MapStr{ "this": "is not kept", "alerts": []interface{}{ map[string]interface{}{ @@ -160,8 +160,8 @@ func TestSplit(t *testing.T) { }, }, ctx: emptyTransformContext(), - resp: &transformable{ - body: common.MapStr{ + resp: transformable{ + "body": common.MapStr{ "this": "is not kept", "alerts": []interface{}{ map[string]interface{}{ @@ -206,8 +206,8 @@ func TestSplit(t *testing.T) { }, }, ctx: emptyTransformContext(), - resp: &transformable{ - body: common.MapStr{ + resp: transformable{ + "body": common.MapStr{ "response": []interface{}{ map[string]interface{}{ "Event": map[string]interface{}{ @@ -246,7 +246,7 @@ func TestSplit(t *testing.T) { expectedErr: nil, }, { - name: "A nested array with an empty nested array in an object published and returns error", + name: "A nested array with an empty nested array in an object publishes without the key", config: &splitConfig{ Target: "body.response", Type: "array", @@ -256,13 +256,12 @@ func TestSplit(t *testing.T) { }, }, ctx: emptyTransformContext(), - resp: &transformable{ - body: common.MapStr{ + resp: transformable{ + "body": common.MapStr{ "response": []interface{}{ map[string]interface{}{ "Event": map[string]interface{}{ - "timestamp": "1606324417", - "Attributes": []interface{}{}, + "timestamp": "1606324417", }, }, }, @@ -271,12 +270,10 @@ func TestSplit(t *testing.T) { expectedMessages: []common.MapStr{ { "Event": common.MapStr{ - "timestamp": "1606324417", - "Attributes": common.MapStr{}, + "timestamp": "1606324417", }, }, }, - expectedErr: errEmptyField, }, { name: "First level split skips publish if no events and keep_parent: false", @@ -289,13 +286,55 @@ func TestSplit(t *testing.T) { }, }, ctx: emptyTransformContext(), - resp: &transformable{ - body: common.MapStr{ + resp: transformable{ + "body": common.MapStr{ "response": []interface{}{}, }, }, - expectedMessages: []common.MapStr{}, - expectedErr: errEmptyTopField, + expectedMessages: []common.MapStr{ + {"response": []interface{}{}}, + }, + expectedErr: errEmptyField, + }, + { + name: "Changes must be local to parent when nested splits", + config: &splitConfig{ + Target: "body.items", + Type: "array", + Split: &splitConfig{ + Target: "body.splitHere.splitMore", + Type: "array", + KeepParent: true, + }, + }, + ctx: emptyTransformContext(), + resp: transformable{ + "body": common.MapStr{ + "@timestamp": "1234567890", + "nextPageToken": "tok", + "items": []interface{}{ + common.MapStr{"foo": "bar"}, + common.MapStr{ + "baz": "buzz", + "splitHere": common.MapStr{ + "splitMore": []interface{}{ + common.MapStr{ + "deepest1": "data", + }, + common.MapStr{ + "deepest2": "data", + }, + }, + }, + }, + }, + }, + }, + expectedMessages: []common.MapStr{ + {"foo": "bar"}, + {"baz": "buzz", "splitHere": common.MapStr{"splitMore": common.MapStr{"deepest1": "data"}}}, + {"baz": "buzz", "splitHere": common.MapStr{"splitMore": common.MapStr{"deepest2": "data"}}}, + }, }, } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json b/x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json new file mode 100644 index 00000000000..2b5fdd89e5c --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/testdata/credentials.json @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "client_email": "foo@bar.com", + "client_id": "0" +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json b/x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json new file mode 100644 index 00000000000..9977a2836c1 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/testdata/invalid_credentials.json @@ -0,0 +1 @@ +invalid diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform.go b/x-pack/filebeat/input/httpjson/internal/v2/transform.go index f4cc7a543ca..d6ca03a84f2 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "net/url" + "sync" "github.com/pkg/errors" @@ -22,60 +23,133 @@ type transformsConfig []*common.Config type transforms []transform type transformContext struct { + lock sync.RWMutex cursor *cursor lastEvent *common.MapStr lastResponse *response } -func emptyTransformContext() transformContext { - return transformContext{ +func emptyTransformContext() *transformContext { + return &transformContext{ cursor: &cursor{}, lastEvent: &common.MapStr{}, lastResponse: &response{}, } } -type transformable struct { - body common.MapStr - header http.Header - url url.URL +func (ctx *transformContext) cursorMap() common.MapStr { + ctx.lock.RLock() + defer ctx.lock.RUnlock() + return ctx.cursor.clone() } -func (t *transformable) clone() *transformable { - if t == nil { - return emptyTransformable() +func (ctx *transformContext) lastEventClone() *common.MapStr { + ctx.lock.RLock() + defer ctx.lock.RUnlock() + clone := ctx.lastEvent.Clone() + return &clone +} + +func (ctx *transformContext) lastResponseClone() *response { + ctx.lock.RLock() + defer ctx.lock.RUnlock() + return ctx.lastResponse.clone() +} + +func (ctx *transformContext) updateCursor() { + ctx.lock.Lock() + defer ctx.lock.Unlock() + + // we do not want to pass the cursor data to itself + newCtx := emptyTransformContext() + newCtx.lastEvent = ctx.lastEvent + newCtx.lastResponse = ctx.lastResponse + + ctx.cursor.update(newCtx) +} + +func (ctx *transformContext) updateLastEvent(e common.MapStr) { + ctx.lock.Lock() + defer ctx.lock.Unlock() + *ctx.lastEvent = e +} + +func (ctx *transformContext) updateLastResponse(r response) { + ctx.lock.Lock() + defer ctx.lock.Unlock() + *ctx.lastResponse = r +} + +type transformable common.MapStr + +func (tr transformable) access() common.MapStr { + return common.MapStr(tr) +} + +func (tr transformable) Put(k string, v interface{}) { + _, _ = tr.access().Put(k, v) +} + +func (tr transformable) GetValue(k string) (interface{}, error) { + return tr.access().GetValue(k) +} + +func (tr transformable) Clone() transformable { + return transformable(tr.access().Clone()) +} + +func (tr transformable) setHeader(v http.Header) { + tr.Put("header", v) +} + +func (tr transformable) header() http.Header { + val, err := tr.GetValue("header") + if err != nil { + return http.Header{} } - return &transformable{ - body: func() common.MapStr { - if t.body == nil { - return common.MapStr{} - } - return t.body.Clone() - }(), - header: func() http.Header { - if t.header == nil { - return http.Header{} - } - return t.header.Clone() - }(), - url: t.url, + + header, ok := val.(http.Header) + if !ok { + return http.Header{} } + + return header } -func (t *transformable) templateValues() common.MapStr { - return common.MapStr{ - "header": t.header.Clone(), - "body": t.body.Clone(), - "url.value": t.url.String(), - "url.params": t.url.Query(), +func (tr transformable) setBody(v common.MapStr) { + tr.Put("body", v) +} + +func (tr transformable) body() common.MapStr { + val, err := tr.GetValue("body") + if err != nil { + return common.MapStr{} } + + body, ok := val.(common.MapStr) + if !ok { + return common.MapStr{} + } + + return body +} + +func (tr transformable) setURL(v url.URL) { + tr.Put("url", v) } -func emptyTransformable() *transformable { - return &transformable{ - body: common.MapStr{}, - header: http.Header{}, +func (tr transformable) url() url.URL { + val, err := tr.GetValue("url") + if err != nil { + return url.URL{} + } + + u, ok := val.(url.URL) + if !ok { + return url.URL{} } + + return u } type transform interface { @@ -84,7 +158,7 @@ type transform interface { type basicTransform interface { transform - run(transformContext, *transformable) (*transformable, error) + run(*transformContext, transformable) (transformable, error) } type maybeMsg struct { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go index 5455d5d7403..6a5867e5bbb 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append.go @@ -27,7 +27,7 @@ type appendt struct { value *valueTpl defaultValue *valueTpl - runFunc func(ctx transformContext, transformable *transformable, key, val string) error + runFunc func(ctx *transformContext, transformable transformable, key, val string) error } func (appendt) transformName() string { return appendName } @@ -107,12 +107,12 @@ func newAppend(cfg *common.Config, log *logp.Logger) (appendt, error) { }, nil } -func (append *appendt) run(ctx transformContext, transformable *transformable) (*transformable, error) { - value := append.value.Execute(ctx, transformable, append.defaultValue, append.log) - if err := append.runFunc(ctx, transformable, append.targetInfo.Name, value); err != nil { - return nil, err +func (append *appendt) run(ctx *transformContext, tr transformable) (transformable, error) { + value := append.value.Execute(ctx, tr, append.defaultValue, append.log) + if err := append.runFunc(ctx, tr, append.targetInfo.Name, value); err != nil { + return transformable{}, err } - return transformable, nil + return tr, nil } func appendToCommonMap(m common.MapStr, key, val string) error { @@ -138,24 +138,26 @@ func appendToCommonMap(m common.MapStr, key, val string) error { return nil } -func appendBody(ctx transformContext, transformable *transformable, key, value string) error { - return appendToCommonMap(transformable.body, key, value) +func appendBody(ctx *transformContext, transformable transformable, key, value string) error { + return appendToCommonMap(transformable.body(), key, value) } -func appendHeader(ctx transformContext, transformable *transformable, key, value string) error { +func appendHeader(ctx *transformContext, transformable transformable, key, value string) error { if value == "" { return nil } - transformable.header.Add(key, value) + transformable.header().Add(key, value) return nil } -func appendURLParams(ctx transformContext, transformable *transformable, key, value string) error { +func appendURLParams(ctx *transformContext, transformable transformable, key, value string) error { if value == "" { return nil } - q := transformable.url.Query() + url := transformable.url() + q := url.Query() q.Add(key, value) - transformable.url.RawQuery = q.Encode() + url.RawQuery = q.Encode() + transformable.setURL(url) return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go index cef04fa034f..12b49af4395 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_append_test.go @@ -129,44 +129,44 @@ func TestNewAppend(t *testing.T) { func TestAppendFunctions(t *testing.T) { cases := []struct { name string - tfunc func(ctx transformContext, transformable *transformable, key, val string) error - paramCtx transformContext - paramTr *transformable + tfunc func(ctx *transformContext, transformable transformable, key, val string) error + paramCtx *transformContext + paramTr transformable paramKey string paramVal string - expectedTr *transformable + expectedTr transformable expectedErr error }{ { name: "appendBody", tfunc: appendBody, - paramCtx: transformContext{}, - paramTr: &transformable{body: common.MapStr{"a_key": "a_value"}}, + paramCtx: &transformContext{}, + paramTr: transformable{"body": common.MapStr{"a_key": "a_value"}}, paramKey: "a_key", paramVal: "another_value", - expectedTr: &transformable{body: common.MapStr{"a_key": []interface{}{"a_value", "another_value"}}}, + expectedTr: transformable{"body": common.MapStr{"a_key": []interface{}{"a_value", "another_value"}}}, expectedErr: nil, }, { name: "appendHeader", tfunc: appendHeader, - paramCtx: transformContext{}, - paramTr: &transformable{header: http.Header{ + paramCtx: &transformContext{}, + paramTr: transformable{"header": http.Header{ "A_key": []string{"a_value"}, }}, paramKey: "a_key", paramVal: "another_value", - expectedTr: &transformable{header: http.Header{"A_key": []string{"a_value", "another_value"}}}, + expectedTr: transformable{"header": http.Header{"A_key": []string{"a_value", "another_value"}}}, expectedErr: nil, }, { name: "appendURLParams", tfunc: appendURLParams, - paramCtx: transformContext{}, - paramTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value")}, + paramCtx: &transformContext{}, + paramTr: transformable{"url": newURL("http://foo.example.com?a_key=a_value")}, paramKey: "a_key", paramVal: "another_value", - expectedTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value&a_key=another_value")}, + expectedTr: transformable{"url": newURL("http://foo.example.com?a_key=a_value&a_key=another_value")}, expectedErr: nil, }, } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go index 00d29d3fa72..37e18bc3a19 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go @@ -22,7 +22,7 @@ type deleteConfig struct { type delete struct { targetInfo targetInfo - runFunc func(ctx transformContext, transformable *transformable, key string) error + runFunc func(ctx transformContext, transformable transformable, key string) error } func (delete) transformName() string { return deleteName } @@ -99,11 +99,11 @@ func newDelete(cfg *common.Config) (delete, error) { }, nil } -func (delete *delete) run(ctx transformContext, transformable *transformable) (*transformable, error) { - if err := delete.runFunc(ctx, transformable, delete.targetInfo.Name); err != nil { - return nil, err +func (delete *delete) run(ctx transformContext, tr transformable) (transformable, error) { + if err := delete.runFunc(ctx, tr, delete.targetInfo.Name); err != nil { + return transformable{}, err } - return transformable, nil + return tr, nil } func deleteFromCommonMap(m common.MapStr, key string) error { @@ -113,18 +113,20 @@ func deleteFromCommonMap(m common.MapStr, key string) error { return nil } -func deleteBody(ctx transformContext, transformable *transformable, key string) error { - return deleteFromCommonMap(transformable.body, key) +func deleteBody(ctx transformContext, transformable transformable, key string) error { + return deleteFromCommonMap(transformable.body(), key) } -func deleteHeader(ctx transformContext, transformable *transformable, key string) error { - transformable.header.Del(key) +func deleteHeader(ctx transformContext, transformable transformable, key string) error { + transformable.header().Del(key) return nil } -func deleteURLParams(ctx transformContext, transformable *transformable, key string) error { - q := transformable.url.Query() +func deleteURLParams(ctx transformContext, transformable transformable, key string) error { + url := transformable.url() + q := url.Query() q.Del(key) - transformable.url.RawQuery = q.Encode() + url.RawQuery = q.Encode() + transformable.setURL(url) return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go index 4d493d241e4..2ad15651f9c 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go @@ -129,40 +129,40 @@ func TestNewDelete(t *testing.T) { func TestDeleteFunctions(t *testing.T) { cases := []struct { name string - tfunc func(ctx transformContext, transformable *transformable, key string) error + tfunc func(ctx transformContext, transformable transformable, key string) error paramCtx transformContext - paramTr *transformable + paramTr transformable paramKey string - expectedTr *transformable + expectedTr transformable expectedErr error }{ { name: "deleteBody", tfunc: deleteBody, paramCtx: transformContext{}, - paramTr: &transformable{body: common.MapStr{"a_key": "a_value"}}, + paramTr: transformable{"body": common.MapStr{"a_key": "a_value"}}, paramKey: "a_key", - expectedTr: &transformable{body: common.MapStr{}}, + expectedTr: transformable{"body": common.MapStr{}}, expectedErr: nil, }, { name: "deleteHeader", tfunc: deleteHeader, paramCtx: transformContext{}, - paramTr: &transformable{header: http.Header{ + paramTr: transformable{"header": http.Header{ "A_key": []string{"a_value"}, }}, paramKey: "a_key", - expectedTr: &transformable{header: http.Header{}}, + expectedTr: transformable{"header": http.Header{}}, expectedErr: nil, }, { name: "deleteURLParams", tfunc: deleteURLParams, paramCtx: transformContext{}, - paramTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value")}, + paramTr: transformable{"url": newURL("http://foo.example.com?a_key=a_value")}, paramKey: "a_key", - expectedTr: &transformable{url: newURL("http://foo.example.com")}, + expectedTr: transformable{"url": newURL("http://foo.example.com")}, expectedErr: nil, }, } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go index 14bccdfb0e0..fcdb1fbbb39 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set.go @@ -30,7 +30,7 @@ type set struct { value *valueTpl defaultValue *valueTpl - runFunc func(ctx transformContext, transformable *transformable, key, val string) error + runFunc func(ctx *transformContext, transformable transformable, key, val string) error } func (set) transformName() string { return setName } @@ -112,12 +112,12 @@ func newSet(cfg *common.Config, log *logp.Logger) (set, error) { }, nil } -func (set *set) run(ctx transformContext, transformable *transformable) (*transformable, error) { - value := set.value.Execute(ctx, transformable, set.defaultValue, set.log) - if err := set.runFunc(ctx, transformable, set.targetInfo.Name, value); err != nil { - return nil, err +func (set *set) run(ctx *transformContext, tr transformable) (transformable, error) { + value := set.value.Execute(ctx, tr, set.defaultValue, set.log) + if err := set.runFunc(ctx, tr, set.targetInfo.Name, value); err != nil { + return transformable{}, err } - return transformable, nil + return tr, nil } func setToCommonMap(m common.MapStr, key, val string) error { @@ -130,29 +130,31 @@ func setToCommonMap(m common.MapStr, key, val string) error { return nil } -func setBody(ctx transformContext, transformable *transformable, key, value string) error { - return setToCommonMap(transformable.body, key, value) +func setBody(ctx *transformContext, transformable transformable, key, value string) error { + return setToCommonMap(transformable.body(), key, value) } -func setHeader(ctx transformContext, transformable *transformable, key, value string) error { +func setHeader(ctx *transformContext, transformable transformable, key, value string) error { if value == "" { return nil } - transformable.header.Add(key, value) + transformable.header().Add(key, value) return nil } -func setURLParams(ctx transformContext, transformable *transformable, key, value string) error { +func setURLParams(ctx *transformContext, transformable transformable, key, value string) error { if value == "" { return nil } - q := transformable.url.Query() + url := transformable.url() + q := url.Query() q.Set(key, value) - transformable.url.RawQuery = q.Encode() + url.RawQuery = q.Encode() + transformable.setURL(url) return nil } -func setURLValue(ctx transformContext, transformable *transformable, _, value string) error { +func setURLValue(ctx *transformContext, transformable transformable, _, value string) error { // if the template processing did not find any value // we fail without parsing if value == "" || value == "" { @@ -162,6 +164,6 @@ func setURLValue(ctx transformContext, transformable *transformable, _, value st if err != nil { return errNewURLValueNotSet } - transformable.url = *url + transformable.setURL(*url) return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go index 31493353b7d..e6afd3dae07 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_set_test.go @@ -130,51 +130,51 @@ func TestNewSet(t *testing.T) { func TestSetFunctions(t *testing.T) { cases := []struct { name string - tfunc func(ctx transformContext, transformable *transformable, key, val string) error - paramCtx transformContext - paramTr *transformable + tfunc func(ctx *transformContext, transformable transformable, key, val string) error + paramCtx *transformContext + paramTr transformable paramKey string paramVal string - expectedTr *transformable + expectedTr transformable expectedErr error }{ { name: "setBody", tfunc: setBody, - paramCtx: transformContext{}, - paramTr: &transformable{body: common.MapStr{}}, + paramCtx: &transformContext{}, + paramTr: transformable{"body": common.MapStr{}}, paramKey: "a_key", paramVal: "a_value", - expectedTr: &transformable{body: common.MapStr{"a_key": "a_value"}}, + expectedTr: transformable{"body": common.MapStr{"a_key": "a_value"}}, expectedErr: nil, }, { name: "setHeader", tfunc: setHeader, - paramCtx: transformContext{}, - paramTr: &transformable{header: http.Header{}}, + paramCtx: &transformContext{}, + paramTr: transformable{"header": http.Header{}}, paramKey: "a_key", paramVal: "a_value", - expectedTr: &transformable{header: http.Header{"A_key": []string{"a_value"}}}, + expectedTr: transformable{"header": http.Header{"A_key": []string{"a_value"}}}, expectedErr: nil, }, { name: "setURLParams", tfunc: setURLParams, - paramCtx: transformContext{}, - paramTr: &transformable{url: newURL("http://foo.example.com")}, + paramCtx: &transformContext{}, + paramTr: transformable{"url": newURL("http://foo.example.com")}, paramKey: "a_key", paramVal: "a_value", - expectedTr: &transformable{url: newURL("http://foo.example.com?a_key=a_value")}, + expectedTr: transformable{"url": newURL("http://foo.example.com?a_key=a_value")}, expectedErr: nil, }, { name: "setURLValue", tfunc: setURLValue, - paramCtx: transformContext{}, - paramTr: &transformable{url: newURL("http://foo.example.com")}, + paramCtx: &transformContext{}, + paramTr: transformable{"url": newURL("http://foo.example.com")}, paramVal: "http://different.example.com", - expectedTr: &transformable{url: newURL("http://different.example.com")}, + expectedTr: transformable{"url": newURL("http://different.example.com")}, expectedErr: nil, }, } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go index f012c33736a..6336330cd90 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_test.go @@ -22,24 +22,26 @@ func TestEmptyTransformContext(t *testing.T) { } func TestEmptyTransformable(t *testing.T) { - tr := emptyTransformable() - assert.Equal(t, common.MapStr{}, tr.body) - assert.Equal(t, http.Header{}, tr.header) + tr := transformable{} + assert.Equal(t, common.MapStr{}, tr.body()) + assert.Equal(t, http.Header{}, tr.header()) } func TestTransformableNilClone(t *testing.T) { - var tr *transformable - cl := tr.clone() - assert.Equal(t, common.MapStr{}, cl.body) - assert.Equal(t, http.Header{}, cl.header) + var tr transformable + cl := tr.Clone() + assert.Equal(t, common.MapStr{}, cl.body()) + assert.Equal(t, http.Header{}, cl.header()) } func TestTransformableClone(t *testing.T) { - tr := emptyTransformable() - _, _ = tr.body.Put("key", "value") - cl := tr.clone() - assert.Equal(t, common.MapStr{"key": "value"}, cl.body) - assert.Equal(t, http.Header{}, cl.header) + tr := transformable{} + body := tr.body() + _, _ = body.Put("key", "value") + tr.setBody(body) + cl := tr.Clone() + assert.Equal(t, common.MapStr{"key": "value"}, cl.body()) + assert.Equal(t, http.Header{}, cl.header()) } func TestNewTransformsFromConfig(t *testing.T) { diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index 8cb8ad041c9..93a84bcc4b3 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -44,14 +44,14 @@ func (t *valueTpl) Unpack(in string) error { return nil } -func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal *valueTpl, log *logp.Logger) (val string) { +func (t *valueTpl) Execute(trCtx *transformContext, tr transformable, defaultVal *valueTpl, log *logp.Logger) (val string) { fallback := func(err error) string { if err != nil { log.Debugf("template execution failed: %v", err) } if defaultVal != nil { log.Debugf("template execution: falling back to default value") - return defaultVal.Execute(emptyTransformContext(), emptyTransformable(), nil, log) + return defaultVal.Execute(emptyTransformContext(), transformable{}, nil, log) } return "" } @@ -64,10 +64,10 @@ func (t *valueTpl) Execute(trCtx transformContext, tr *transformable, defaultVal }() buf := new(bytes.Buffer) - data := tr.templateValues() - _, _ = data.Put("cursor", trCtx.cursor.clone()) - _, _ = data.Put("last_event", trCtx.lastEvent.Clone()) - _, _ = data.Put("last_response", trCtx.lastResponse.templateValues()) + data := tr.Clone() + data.Put("cursor", trCtx.cursorMap()) + data.Put("last_event", trCtx.lastEventClone()) + data.Put("last_response", trCtx.lastResponseClone().templateValues()) if err := t.Template.Execute(buf, data); err != nil { return fallback(err) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 6bb66fddf54..d60c4a81718 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -19,8 +19,8 @@ func TestValueTpl(t *testing.T) { cases := []struct { name string value string - paramCtx transformContext - paramTr *transformable + paramCtx *transformContext + paramTr transformable paramDefVal string expected string setup func() @@ -29,21 +29,21 @@ func TestValueTpl(t *testing.T) { { name: "can render values from ctx", value: "{{.last_response.body.param}}", - paramCtx: transformContext{ + paramCtx: &transformContext{ lastEvent: &common.MapStr{}, lastResponse: newTestResponse(common.MapStr{"param": 25}, nil, ""), }, - paramTr: emptyTransformable(), + paramTr: transformable{}, paramDefVal: "", expected: "25", }, { name: "can render default value if execute fails", value: "{{.last_response.body.does_not_exist}}", - paramCtx: transformContext{ + paramCtx: &transformContext{ lastEvent: &common.MapStr{}, }, - paramTr: emptyTransformable(), + paramTr: transformable{}, paramDefVal: "25", expected: "25", }, @@ -51,7 +51,7 @@ func TestValueTpl(t *testing.T) { name: "can render default value if template is empty", value: "", paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, paramDefVal: "25", expected: "25", }, @@ -65,7 +65,7 @@ func TestValueTpl(t *testing.T) { name: "func parseDuration", value: `{{ parseDuration "-1h" }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "-1h0m0s", }, { @@ -74,7 +74,7 @@ func TestValueTpl(t *testing.T) { teardown: func() { timeNow = time.Now }, value: `{{ now }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 13:25:32 +0000 UTC", }, { @@ -83,28 +83,28 @@ func TestValueTpl(t *testing.T) { teardown: func() { timeNow = time.Now }, value: `{{ now (parseDuration "-1h") }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 12:25:32 +0000 UTC", }, { name: "func parseDate", value: `{{ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 12:25:32.1234567 +0000 UTC", }, { name: "func parseDate defaults to RFC3339", value: `{{ parseDate "2020-11-05T12:25:32Z" }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 12:25:32 +0000 UTC", }, { name: "func parseDate with custom layout", value: `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006") }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 12:25:32 +0000 UTC", }, { @@ -113,7 +113,7 @@ func TestValueTpl(t *testing.T) { teardown: func() { timeNow = time.Now }, value: `{{ formatDate (now) "UnixDate" "America/New_York" }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "Thu Nov 5 08:25:32 EST 2020", }, { @@ -122,7 +122,7 @@ func TestValueTpl(t *testing.T) { teardown: func() { timeNow = time.Now }, value: `{{ formatDate (now) "UnixDate" }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "Thu Nov 5 13:25:32 UTC 2020", }, { @@ -131,34 +131,34 @@ func TestValueTpl(t *testing.T) { teardown: func() { timeNow = time.Now }, value: `{{ formatDate (now) "UnixDate" "wrong/tz"}}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "Thu Nov 5 13:25:32 UTC 2020", }, { name: "func parseTimestamp", value: `{{ (parseTimestamp 1604582732) }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 13:25:32 +0000 UTC", }, { name: "func parseTimestampMilli", value: `{{ (parseTimestampMilli 1604582732000) }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 13:25:32 +0000 UTC", }, { name: "func parseTimestampNano", value: `{{ (parseTimestampNano 1604582732000000000) }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05 13:25:32 +0000 UTC", }, { name: "func getRFC5988Link", value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, - paramCtx: transformContext{ + paramCtx: &transformContext{ lastEvent: &common.MapStr{}, lastResponse: newTestResponse( nil, @@ -169,13 +169,13 @@ func TestValueTpl(t *testing.T) { "", ), }, - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "https://example.com/api/v1/users?before=00ubfjQEMYBLRUWIEDKK", }, { name: "func getRFC5988Link does not match", value: `{{ getRFC5988Link "previous" .last_response.header.Link }}`, - paramCtx: transformContext{ + paramCtx: &transformContext{ lastResponse: newTestResponse( nil, http.Header{"Link": []string{ @@ -184,7 +184,7 @@ func TestValueTpl(t *testing.T) { "", ), }, - paramTr: emptyTransformable(), + paramTr: transformable{}, paramDefVal: "https://example.com/default", expected: "https://example.com/default", }, @@ -192,7 +192,7 @@ func TestValueTpl(t *testing.T) { name: "func getRFC5988Link empty header", value: `{{ getRFC5988Link "previous" .last_response.header.Empty }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, paramDefVal: "https://example.com/default", expected: "https://example.com/default", }, @@ -202,7 +202,7 @@ func TestValueTpl(t *testing.T) { teardown: func() { timeNow = time.Now }, value: `{{ (parseDuration "-1h") | now | formatDate }}`, paramCtx: emptyTransformContext(), - paramTr: emptyTransformable(), + paramTr: transformable{}, expected: "2020-11-05T12:25:32Z", }, } From 92fadf8e93f8bc0023d29d018fe2b564437f5356 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 30 Nov 2020 16:53:57 +0100 Subject: [PATCH 32/35] Add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2ba1c3670d4..61ce175cf94 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -742,6 +742,7 @@ from being added to events by default. {pull}18159[18159] - Improve panw ECS url fields mapping. {pull}22481[22481] - Improve Nats filebeat dashboard. {pull}22726[22726] - Add support for UNIX datagram sockets in `unix` input. {issues}18632[18632] {pull}22699[22699] +- Add new httpjson input features and mark old config ones for deprecation {pull}22320[22320] *Heartbeat* From 42fb48527dd74b327202b96976996c28d5b6eee0 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 30 Nov 2020 18:09:58 +0100 Subject: [PATCH 33/35] Do not copy mutex on delete transform --- .../input/httpjson/internal/v2/transform_delete.go | 10 +++++----- .../httpjson/internal/v2/transform_delete_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go index 37e18bc3a19..c8c54b8141e 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete.go @@ -22,7 +22,7 @@ type deleteConfig struct { type delete struct { targetInfo targetInfo - runFunc func(ctx transformContext, transformable transformable, key string) error + runFunc func(ctx *transformContext, transformable transformable, key string) error } func (delete) transformName() string { return deleteName } @@ -99,7 +99,7 @@ func newDelete(cfg *common.Config) (delete, error) { }, nil } -func (delete *delete) run(ctx transformContext, tr transformable) (transformable, error) { +func (delete *delete) run(ctx *transformContext, tr transformable) (transformable, error) { if err := delete.runFunc(ctx, tr, delete.targetInfo.Name); err != nil { return transformable{}, err } @@ -113,16 +113,16 @@ func deleteFromCommonMap(m common.MapStr, key string) error { return nil } -func deleteBody(ctx transformContext, transformable transformable, key string) error { +func deleteBody(ctx *transformContext, transformable transformable, key string) error { return deleteFromCommonMap(transformable.body(), key) } -func deleteHeader(ctx transformContext, transformable transformable, key string) error { +func deleteHeader(ctx *transformContext, transformable transformable, key string) error { transformable.header().Del(key) return nil } -func deleteURLParams(ctx transformContext, transformable transformable, key string) error { +func deleteURLParams(ctx *transformContext, transformable transformable, key string) error { url := transformable.url() q := url.Query() q.Del(key) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go index 2ad15651f9c..22cbce310f9 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/transform_delete_test.go @@ -129,8 +129,8 @@ func TestNewDelete(t *testing.T) { func TestDeleteFunctions(t *testing.T) { cases := []struct { name string - tfunc func(ctx transformContext, transformable transformable, key string) error - paramCtx transformContext + tfunc func(ctx *transformContext, transformable transformable, key string) error + paramCtx *transformContext paramTr transformable paramKey string expectedTr transformable @@ -139,7 +139,7 @@ func TestDeleteFunctions(t *testing.T) { { name: "deleteBody", tfunc: deleteBody, - paramCtx: transformContext{}, + paramCtx: &transformContext{}, paramTr: transformable{"body": common.MapStr{"a_key": "a_value"}}, paramKey: "a_key", expectedTr: transformable{"body": common.MapStr{}}, @@ -148,7 +148,7 @@ func TestDeleteFunctions(t *testing.T) { { name: "deleteHeader", tfunc: deleteHeader, - paramCtx: transformContext{}, + paramCtx: &transformContext{}, paramTr: transformable{"header": http.Header{ "A_key": []string{"a_value"}, }}, @@ -159,7 +159,7 @@ func TestDeleteFunctions(t *testing.T) { { name: "deleteURLParams", tfunc: deleteURLParams, - paramCtx: transformContext{}, + paramCtx: &transformContext{}, paramTr: transformable{"url": newURL("http://foo.example.com?a_key=a_value")}, paramKey: "a_key", expectedTr: transformable{"url": newURL("http://foo.example.com")}, From f807f73d9842012a7f2403e77801182e4ef72c22 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 1 Dec 2020 10:42:55 +0100 Subject: [PATCH 34/35] Add PR suggestions: - Modify context done error log - Change is_v2 config bool for string config_version - Change retryable logger to be created from input logger --- .../docs/inputs/input-httpjson.asciidoc | 32 +++++++++---------- .../filebeat/input/httpjson/input_manager.go | 2 +- .../input/httpjson/internal/v2/input.go | 10 +++--- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index dbcdfbf00eb..cec658e0bcb 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -22,7 +22,7 @@ Example configurations: filebeat.inputs: # Fetch your public IP every minute. - type: httpjson - is_v2: true + config_version: 2 interval: 1m request.url: https://api.ipify.org/?format=json processors: @@ -35,7 +35,7 @@ filebeat.inputs: ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 request.url: http://localhost:9200/_search?scroll=5m request.method: POST response.split: @@ -60,7 +60,7 @@ Example configurations with authentication: ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 request.url: http://localhost request.transforms: - set: @@ -72,7 +72,7 @@ filebeat.inputs: ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 auth.oauth2: client.id: 12345678901234567890abcdef client.secret: abcdef12345678901234567890 @@ -174,7 +174,7 @@ What is between `{{` `}}` will be evaluated. For more information on Go template Some built in helper functions are provided to work with the input state inside value templates: -- `parseDuration`: parses duration strings and return `time.Duration`. Example: `{{parseDuration "1h"}}`. +- `parseDuration`: parses duration strings and returns `time.Duration`. Example: `{{parseDuration "1h"}}`. - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `{{now (parseDuration "-1h")}}` returns the time at 1 hour before now. - `parseTimestamp`: parses a timestamp in seconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732}}` returns `2020-11-05 13:25:32 +0000 UTC`. - `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732000}}` returns `2020-11-05 13:25:32 +0000 UTC`. @@ -193,11 +193,11 @@ The `httpjson` input supports the following configuration options plus the <<{beatname_lc}-input-{type}-common-options>> described later. [float] -==== `is_v2` +==== `config_version` -Defines if the configuration is in the new `v2` format or not. Default: `false`. +Defines the configuration version. Current supported versions are: `1` and `2`. Default: `1`. -NOTE: This defaulting to `false` is just to avoid breaking current configurations. V1 configuration is deprecated and will be unsupported in next releases. Any new configuration should use `is_v2: true`. +NOTE: This defaulting to `1` is just to avoid breaking current configurations. V1 configuration is deprecated and will be unsupported in next releases. Any new configuration should use `config_version: 2`. [float] ==== `interval` @@ -275,7 +275,7 @@ Can be set for all providers except `google`. ["source","yaml",subs="attributes"] ---- - type: httpjson - is_v2: true + config_version: 2 auth.oauth2: endpoint_params: Param1: @@ -348,7 +348,7 @@ Defaults to `null` (no HTTP body). ["source","yaml",subs="attributes"] ---- - type: httpjson - is_v2: true + config_version: 2 request.method: POST request.body: query: @@ -430,7 +430,7 @@ Can write state to: [`header.*`, `url.params.*`, `body.*`]. ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 request.url: http://localhost:9200/_search?scroll=5m request.method: POST request.transforms: @@ -454,7 +454,7 @@ Can write state to: [`body.*`]. ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 request.url: http://localhost:9200/_search?scroll=5m request.method: POST response.transforms: @@ -569,7 +569,7 @@ Our config will look like ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -660,7 +660,7 @@ Our config will look like ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -730,7 +730,7 @@ Our config will look like ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 interval: 1m request.url: https://example.com response.split: @@ -774,7 +774,7 @@ NOTE: Default templates do not have access to any state, only to functions. ---- filebeat.inputs: - type: httpjson - is_v2: true + config_version: 2 interval: 1m request.url: https://api.ipify.org/?format=json response.transforms: diff --git a/x-pack/filebeat/input/httpjson/input_manager.go b/x-pack/filebeat/input/httpjson/input_manager.go index 1efa41470b2..68a929fb8fb 100644 --- a/x-pack/filebeat/input/httpjson/input_manager.go +++ b/x-pack/filebeat/input/httpjson/input_manager.go @@ -43,7 +43,7 @@ func (m inputManager) Init(grp unison.Group, mode inputv2.Mode) error { // Create creates a cursor input manager if the config has a date cursor set up, // otherwise it creates a stateless input manager. func (m inputManager) Create(cfg *common.Config) (inputv2.Input, error) { - if b, _ := cfg.Bool("is_v2", -1); b { + if v, _ := cfg.String("config_version", -1); v == "2" { return m.v2inputManager.Create(cfg) } cfgwarn.Deprecate("7.12", "you are using a deprecated version of httpjson config") diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input.go b/x-pack/filebeat/input/httpjson/internal/v2/input.go index f63b3769436..8ec7d1a6679 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input.go @@ -28,7 +28,7 @@ import ( ) const ( - inputName = "httpjsonv2" + inputName = "httpjson" ) var ( @@ -42,9 +42,9 @@ type retryLogger struct { log *logp.Logger } -func newRetryLogger() *retryLogger { +func newRetryLogger(log *logp.Logger) *retryLogger { return &retryLogger{ - log: logp.NewLogger("httpjson.retryablehttp", zap.AddCallerSkip(1)), + log: log.Named("retryablehttp").WithOptions(zap.AddCallerSkip(1)), } } @@ -136,7 +136,7 @@ func run( return nil }) - log.Infof("Context done: %v", err) + log.Infof("Input stopped because context was cancelled with: %v", err) return nil } @@ -157,7 +157,7 @@ func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSC Timeout: timeout, CheckRedirect: checkRedirect(config.Request, log), }, - Logger: newRetryLogger(), + Logger: newRetryLogger(log), RetryWaitMin: config.Request.Retry.getWaitMin(), RetryWaitMax: config.Request.Retry.getWaitMax(), RetryMax: config.Request.Retry.getMaxAttempts(), From d4afbda13d8a163ffde737c8749118b78948fe45 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 2 Dec 2020 11:20:06 +0100 Subject: [PATCH 35/35] Documentation fixes --- .../images/input-httpjson-lifecycle.png | Bin 0 -> 52978 bytes .../docs/inputs/input-httpjson.asciidoc | 210 +++++++++--------- .../httpjson/internal/v2/input_manager.go | 7 +- 3 files changed, 110 insertions(+), 107 deletions(-) create mode 100644 x-pack/filebeat/docs/inputs/images/input-httpjson-lifecycle.png diff --git a/x-pack/filebeat/docs/inputs/images/input-httpjson-lifecycle.png b/x-pack/filebeat/docs/inputs/images/input-httpjson-lifecycle.png new file mode 100644 index 0000000000000000000000000000000000000000..b60d80fbb8c0b9f09661432875535e0878c02816 GIT binary patch literal 52978 zcmb@u2~>@3+c$nCl1xP@Lo-Ryqzq}2CW!{6B&Cv6lF+D$MkN^<2+e7tfhK7n6os@k zP?RFgn&;{NJ8?hX`+m>+{MYxb_3d>(>$z`g?`vPzc^=1aI?l^l2UM3btzn`lYUy4z zB^`>IA45@e{frFwlk0u#J@~)*=M?wqGUDaNcsv0Az37bE;d2zV>?QdhU6}B)wfMso z=ami4AGAAh-pTZ=1!duCX>~%x+|D-U#bX1C;-U5`?bda^+w;}Mh^=-`zMp+cK=2M@ ztDkzLm0tzlK0c=OL;?1(w-WVsakp&(O)V}j9^HGeGUvav_#>bpSo@+PAUKM7uK&R)_?vYh3&uoLP4ASNh_;by`5cbyh=yeFG@03 ziKZ@(%5gWJ*|7gv_tO(*Q%bsHU8(ME6Pi??UALjbsQiWh@*$mvBnOmde6O)+xpEoT z3=tvU@7%V(-f8A$^ku{ir!nfu`oVhnAn`dg%TSuIm5(YvkNEFf-%( z_4R3zWmjlW+(Sep0co5aN)v*j*gBR)07(iZQE8Y=aO;mntp$8 zSAAt~(^DtW19x{^yfF65wCu|2GY$?9yZLI@it5bhz4BhToO$h9VfWme9N&izw`6B$ z*W75H*97)H5799?z)$+UdcKX-@C_o z%GNg8{Ic%N&`_Ton|i)Yn-*QYdetENr1nwv6rHhOU-wvB3mLnQ=vf(hc*xC;e_ww5 z`0*H*p`T~$?Jwe`-pR(w`Ym{S(>1va!yG2jlIgR)l}xyd z%TZnd0fy_>uZxL^DI^`Z_wCy^t$Jme^MVBn#(VRcuid*BjJp%>&|T+nG+}aoD80h8 zjZTh6l+58o<1O;{&bU%jLB8H7GJ(zjlqM;Cv_FuP;vycAj-~bZqUYj<*(8 zwQzQ?KNZGlk}E&A!)-kC{1)3!nXwel8p)G9`;X+FI((E}`?8S;JMO&MhnU^1ckGH; zrpEg4@p_whcoeRzky4*rw|e!aNGThE$nAC_GBPqgVPRpJj~^f0S=8EUWd^jq2%sSy_>%@quQoyT=47^O-xMk z3km|Sa_(X}dh}?P>qwX3X49F25o3AyDv^;w>(;GP`d${Sj0ffu6%}Rh)X_HQ?f&O>`?sVu9H0@Z~sL)H%u z)-M&b%5BJ3IQ{vNG9#~&v2hf~wo|vnwr*Yd`Sa(T!*AP9;Brgr>+@(9KdYie@zGHS zFGqcyo*1&3@k`IG^`}AwYMUbk7b_$tCNARjdHwn|^RHjO;w&n|6-6TI?K@NZ%7WK2 zH6G+1m`wkyyGS8xMmsbrN`$JJQZPEa)7;#=yFgIri9>hzhuT^;&$*dA$#oZ>5FH)+IYOU`gsR#H0?+j5X^Ek0%BeoHmi_oc(dQxFmdHq~63AH!k4H zmC*YeHf{_;m1__(F4C>1OV*B8x$M~6aJjHhhS!JL#BIP=Oj43{_3G8N>g`{ioSWz6 zjEzX{*U~kT%u}PiX*t7D7IU)o?XBr1CChGW`0a}jiS?YD#r{@SQ)5)GH15KxdWIS> zFxqJM;`8U#D>oi^d2g3f!o|`-Gjs44<2lO zeOvoTk|uV>E`WSsjc{dA-2_brbf=MfWQ#&hfVn6w<%oS12>tj4SUHNwG;80dpX6jxi=f;6O z;Cu7tN;h}+Ec@>&R!zA7yf@$USBjSK2nnf6tCmdtdTQP4-I!;$N8l!}oE+E9jhZT0 zIbzQJ#}X0}C~CB?ImGmZU+j^*(`U||dx8CQJvv(Sp=bZM-EX60*Op8VE;ja@ad38V zG3{$AVsiQM^JnXeQt1e87COVKyTd7gt9^jS|1pCLvRGppkCFlBEUX1s@Se|A2f+0UY|JXAxIwl8pLHB7}yk-C3;H8WV z(L)am=$;sfylZ}BvAB48aNnth>_w+fpYEtlV=pZ&Wu;+-Kf{w~>FK$G+GyVT^0InJ z^xW`AO%Cbvf_LxT6Ze>QMlqIg@ci`Y)3M3nPBje;B~)v#)NWB~8egV=WW@LB(_N## zz6RaiYS~&PKNtEl~qE=!cbET3NXmH(-2rs*jG^v}x0LM|{Y)wzdTuH*UnL6u#Stm3QsU@g44yk}4rw z!RbeznswErU|(I=**P;kO)j?Beafsc*Ope8kgy&XNFFOi7C-*ox4iGi*`11e_Ox_& zFV@u5d>bjX7~O=)W&Qf~3Rs^zcdo|jDb{Vea%D+RQ_$ac%a_uMJO^jaJA*K+IZ8mip zompe44gM9sGSE4VTfR>pysz>6I=^Z{6$-`keEaX;T3aunTMB+*kGL)4B7tR@Y4(bV zESAQ`wIws7cLsN-l>0D$%CqA}TlB^4qNuVE?vS`RJ}b}ASMn8kBWSberkvEbuG%y_ zyj${~a$d9+CR8f{jaxihGAc$a#`{|W0t0WzMdPj>Oga!Iw{ZS^JdxPUOlEAUur7no z?(IXmY32%gZP*B(9$PMd^yrb8MFlI0$g!VqZ+FzBtel>nrYO~`>z?1=e@D#n{W?;{ zK5ELp3Si+s)B17|rC?;V?&`Xom+;;9c5=yktlPG28y*W~n0I=qr>Ey#X$K}03~b+p zl;r6z<1VL#t~a2<(^C%~KBNm}*oT(;t)nC0$+>nuv@Beuvc0`Hxw#Z06H_TFA8Oej$6{NndT)EP1@XbawHMM8BPAb&+Ji~IX zD~bCeO5eO$^ybZ*fU8%-X%~8P>W^7l2jV8aEHB^Je$pX2;(cZ1izwL(*Fr<*Z{NP1 z>_PnuV`X)9CO`r|Y)?{qICeR19vm9VI$6!Pe!~XS%J40edi3=SLurv*Q27YhHp7Ck1BBNMeScK<2uaf^y|y@ z+*eIaTne50ol%6V@9$r`*(je`TU(n$+-$za{{4yDmQx}}3w#0s7(+utkI@qLeyFdH zIL$TI+vtzN`@FfCn=Gr)O$Wt}J)^Me_Em_Yi^$K8^M{KZW!k!Rt5L!EMSJ(|#bZ2} zGuk;cym`cFq^s5-%WMI5tEsuU_v_bcxDzkFe&wXj4E@~iHuP>g;OBGn3;Wq=mnEwu z=6lYJup7Iyu~E-HWtrQ5FI(z1JJJ67&70?MZ*SE$FbKfDp`+sB;!u+1V>F}XZf&>Y za{TpW5gz^ejT>Kl`ow|nSB%>bzWMMa+<3CV$NHN66B9R3-=6jr?>Tmi2M_Sd!1r=< zH1GcY{%1hTQPMxl(RJt)3NEIk&PK_&=uAq~r5xPoHt>p_JaXK}z|&s~=mZ}|hlhuc zcco>3CdGzYDk;tPm}p}!ocMW%?1yc3O;=D7`FMHhCMPGIr^m$)+>>5_j)sT92iV+K zG{%L+B#yQTH2eZL)XK_ghwHE)!?JaZ=gyr=%-Y)IHnd;HW11sgHKgp8xOwPlza)*@ zmohWOe0+Sq;i7NtJSPC0n|Z!dhay*=={jQVeBpvQ5W|i$->7->=3TpccL|<3oAui3 ze7W*^yUzVw;xRj6+0of)`X%4~T4h_lL-#7`8&*2Gt#dQYbI-qgS?9lu^HM|vyP3JU zsg2FzCq`FgFN}Vx2oolsH#yoHgwktpHJgjc_^qlJX{CFXH z19<|#Zf7P2Exr`HudUB8DKBv7&a|jpr4cE)$h_>zbJQDxoW73_d?y=y#flXlGEC%2 zBL{3$)6^`-wiEhd3552k&}FN~%!s~KWNz*@!|an6Yt!`O&$Yil`SE@*&Gq)}+oa^2 zZYsPCMngXAa= zT(CHxP0!b->`hHgw4L!4WA%8Fq}Fe>eE$*)oMqK227P^fB~{fIb#<#A8s;dTI<RNFNJuNw1pUzo4GkOHKWG~bCqa_}k&2B?W<)-`hP>FP5sHNY* zy~cM%V*6-i#$NNn;1wYFD@$4e~87C*z`t+l8)QJ-(x;{yTrCG732FmoZ z6r$cPqFM>8lN1#d6}M}W={$F5o9)YMynE2AGK|@%m*}oO%!&^HZ?bJ05Al8la2A%$ zd#~D`T%59ZMyQ)_^UXIe*-hMdJ$roOpYScgtfyH&Z*ijQ*Z}tXgIC`9#UL zZ{NNI(MyGI@atN@zdYWkCgq+>b%a6NsFOKXzS{(_hD`Y0-WbnIw zCZ>MInpk?}&v$oxv$C=zJ$}R~2eFpj+`M@+tE;PPD=s4J zr#@FkU*y9_-z!H|vt~9OtmStuE-sF<`gmU`S|gx+8kX>sf~-4 z)-?{vlh>HzhR2I>t%{7iipS4zHD)Yp$p&vvLQMXvnn{Mrn!BimE6s!R-hn$lTO0giBTlSBoqC z6tZ^v0SIb~fJ3U2G`X`bjGgwC(X--CEhs?&tbcWP=edL`RNO4kv72(sx#f(;Cnh#M z*lJxPnD}VBU6WD%)2H>$9Ty7R6XIu?5-F5&Q&=Jb#Fg9oubg$=m|{6`g7N= zUHj&oIzG^;sHo)Q6BKJgJCs9j-3mZYR#H}e4w$rf)n>q06_rw;g4heLii-2l9(a8e zz2$Hattm(#D8C}JZPrDYq$G>6agqzC+=e>*g;NC z4y0SJ@s3z!$?E%spgEgOiq`?)2yWi|66(qIhzS3Uno$aPH|oP9i+!60GvedvZES3O zM_+?8p-PN*XOw)%vopcXdR|th()Q?UzCAs}9LP`+_i;JjK&P|}>)yTVO_7z_uc46; zB`zQ!kooA*Jgvzc{$XKkK(AsbYJei~CM6#ILqmIpe!kNaX&=(LIhmH2xBx=U70z9& z@Z?{iaeV9U4gzHo`hlJ+JfcAF1&q-GFhr?eD547-X#%MwCHuq)fk;WKd6e@+TSRQy z5m%S&%uFw+7dl0d!I<#mmVwicPfn_r?~s+{1kiil>^T>_^W0ChZb*@W!}I5ReSU1Y zM^$wpsywOJHH+sZ7t9)$cpQzjaR4P0p3ZqJlug;yuU#7ijmkGqr650_M_!&AQ12uf6BT3Im`lsJ$iTpmnVG3o;KY-=m8 zs45$_D^p2n#3m7uvdNKd`)|+aQpSFw44#>rb@y$n7%QC_n<)nBKn14*SzWS5l94<^ zN5?}GwlR<_UjUP_jas6U#Ky;auibw7TVjN=MLh&DZsBe{0X>9pdQ$h5sv)bzm+4LseshYuens`g`9vUR0~hp&QGab{-HA*_;P z_3A|j4<7VAw`==$ed#=u7NQRjNPDwv!kN?is-eJwb!B?jJ5o)&iaPgD$n8mAtN~CVJvan}e+CWsA`W}aa0Pxp~7 zJwm^LV+cyiG<{)h!!DR2cyb88j*?D~H#R6ZzX{5dukT`Zw3Y?^&9?7K8sh=^zC&H~ zPf6KCx^(vGuWKyd-_?F{YGPtSed^Qe*U6=$4<4QF2c@P3XXWPR(iYgE!|ibH=OH}k zRS$zaOq$D3wwIL|=tQRvo)%;R7N~7Y*N_eNvfgYQ~DA<@4h`(GahXnoP7+ zhV0kTD{6GoccriGZAq&Yg@uKzw3+WA@&wGu&rPoc z>rzIKPLNDZEzyjUxd=^7(X}mapWa+C7yY%|5ilBOkQK6sM!M_Gu{I&OY5H!zUX|Ko zl$?At^}RT6D$}1;ulZDDi*X?b-trahT$V%kA)-Ry`+#X?m7blOop;D%m-P7#FTf8q z^V=Zd>fdEk;^K124n?(BH{Y+9yQMATp~YNxXXoDXqP;f-^3S$0;Rbp4_n#r~4wypU z+OsbGC~59X*6qkFbsHWYCglWJm`_NE3HRhv(S^KHiMi7Yg&UUXghb{p*Q<>W{IxJ` z(&LCRC@&Ke(}#R}bKkS`=gNNzq`3bd2l{NhvBuA9 z2&t4?UoaKz{FCAOtE4p5)o+yX2+GH7(;As*1CgJecGI{gtvBhL#0TBxB{XVM zfY-@5Geg%=Qc>|6UM#9LX>s^y`$=70RvL}g52&oZ)Nw+5wJro`Utlp)Nc9Fqu2Mw* z#5RUHnwsr_x_uq(3JQE@Jq@~Mta8vQwx4<{i(E$(GmMLv*Q^PIQf%7WkfXkR!^o@0 zHO&&JR;pMTL_olDH3#)&qBpyrloU^@V)Z^XwQK3z@9u0@#ofPg^Jb!bVQXt^3`$IY znvULi2M2MpmyB09#OLAJ5qXkmX`1pgtY8I~($hr%N-jhF*F9Gis*^Y?y<>;qwAtB^*6h7d?lXI4xo)lZuMkOiKWCyIy-sTc+ zP@fk7-Bf6?8jq}OQA1}$a7g}{7DZB+P!7sd_0kfuw1vz;2GQ?fGVPW1$Ia6+Ffiz@ z|AYcz2F7^{vi18YSt7!{eECvMRuZ5y%eq!*^WjHIiEf*>Y^lbMOWd{#iZT2o3F=kB zK^0BS0_h=Jrd|1=t9Kv0l3^E?xei4Abu5_fZ%xOQ$xNJ`pg$91^Gqlnfh??t;a z1au4c=CMDk)p~H8^L_6W4Zo}L~ejHO(!eE+@-M&O1m=LTAped$8BN=SfyCG>Nw(Q>`2qO8ojZCh@x zq$+KRJJ)_G13EcEi^vP?}!Pu}}A37>KO=1u;QVVzeoyS@9rJ*Ov{3#Bf)3U1YI z1qFqiN}b!liQ2A9;*ygYv3kAl-8=g5r*>oN5zmV>QCSQ28f<~%78aXqEpdrN>W(>o z>@5n=K5rMtPTB%|&MOd4Z5^FUa7+NWjMnsPv*A*(XOwsErt_jjEk>Wv**QEiLRR?O zd%N`4cn!X}xsfoZVa5Y`x+x`n#Zw}r9p9C6U$b-P&Re^j(rNs>QWjQLm!WX)4XK@; z9;#w1j=c}?4>#r|nCk%OsC66Y1|w8ux6#j*f{rqeN{c@&`1EEk(((Zabm0gn*9I3C@U5fsn&_GRPSm&)|@< zW+ggSN5@IH0N-wE%3Xv*$@=5mnKNGKio>H_kT5!{dj<`2Pc7o)> z^@yveN$?0;dLtilk#8rk*9vf8J_!jOM&6ep^0R{2MgO?SWnhBz3sh89gYa~MKx>F# zNpvOQC5j?Swe9uTn#w?$zUk@dCus?nao1MCD56c{;VlN%GyQ|Y`B2q_*Ks_FiBVX> zDXmLPZOUx$#~by^WC=zWD099;mQ_((P|kht-Mg2Wo4YEAU33+dK&Kb6<%odsgD_ce3n;4(gQ#renV4EyZX}j8WXsfS6YGZJTK~yi zwhfXX-NL6%ol=6o@CrqBbGmVn&^r?d3Irm-?C~ddJ1)o!sEMimE<~yT+;VcGOd*?+ zi$_ZodI6*Ug*o5=dIi6OAIV3aZu$kd!!pgi=sRqOy=8rtYG*jnPG2J+On87zBNt9ST|L`}YT| z`g-CBZ1JRk1%1H1P9mF7O7&WJUn8#2t=qR3p_LQ4!1u!A!N!l!x5>I9 zl1px4Ty9-s3y1LP5ebhCk9$y(JGl>@Na7J zoO9pw4yrI&Q$T|xR)Dg7+SkIhVXPiBQQW+2iRsCcf>4f#(|)ocJ7ve&9|6pY0jM9< z1r9cJ3b$_EVx&qxemqFr5vydHoUViC7Nt&EH+(&1d{x5P!+v_Aw$q9-pb~*+uRvGP zGq?{Izzlf?&3AkB99%qweVCzeuo5|iBHtTACJXVjN+lpMFjGEI>EP57pHCNMB!K2f z!-j);vZ-*hLRfbtsF@GmT})h@1?Y?&_c!ef^B}oO5RTZ443oAXOWlFZQ<~j#8H@@p z)BXh?a&3>ejkoxdTB&pJAgq)-OyxOh>gt!SUOkXguth&n?v%V$di&uLf;T&=`Lj<} zD|SZcp*`H$ZWo7Ls`ya(2qAOf$1^2+kp@B8AgG9n9?3)Mr1s5`ej)P{Cr010nDV7^0i2*A%_W&!RM##0@S)##Q9?!A z*;Ah~>9BBiKR)q}dG+ec_%!ixC0$kHgi-%lEOGxe3!=Sf7n7pvj$92;`1{8?Q|0j58CGubY@;`paf4ZtveL6w#8IZs_2l0SpF0(0x z$SWgv0=k5AHI+u#t2E!;zg(`rkCC^lfY|~$J&GDhR#Vc|%yj43UpBd4X@sqDW87p= zd(w%IFTWqj_%CJyYo#o}_{Cs@#UcGu{ zy>|{q3dhc~35K+E(I)5bOUXWj!dHe)*j=#WpTMZf8a6NK3!@OoPF_T@j031A@j1i6 z)iCskxe6evx5E>W6UJrBJbn&8e)Q-J_PeF=9LyX-h5>@2cP@3fgZ$%(s;FRKhVQ67 z4oBDhv`nG|l{xx)`b8o*Zn2v^?euZn-Xu1st-rii`B)adBYwzgM4b+Uu`H+8$- zbGU;AzYjf(y!X)Ps0xUEOj_EJdS%|f?<@5&nbHNW>?Z~!5g(6m!3pw4Tm-?Dhpb)z z!K;ygK;q3C4Flm6{6pHjP@+3hAXfJRxt>4un32Yh5N92<3yTaec5b6I_Gt~-z3tie9xuqtTc1~#V zEw%b43n=hZPJku_6%_`6*OtD%_&KAB_ z)>RTz6@1OlvZ5WR)0z0fT3Rt;GYChLL=@QMa;VkRGnfU-mM>pU$S3V=dsVbQ;%so_ z=HnLlN)mtsua6!!+0*Qjjr${gi06$ssbM*m7d+0CeBWKf6 zd)}R83wsVR3GUu4*$a-rgD7q5FI~Fyp~TaZXdp6*PCs6=AP4v@K1A-ueT(DAb#GnG z$`VHoX}ZP@oth{tpmIW2EdGg~&vC99SylKa0>lp?OrbO8D>NmdXd3(i?}lZfpr#g3 zVdW2JqB>r69P|UbPjqGiB1X~Ds^`z?U}o>Z4STn}Yc=o?VT{OWMe8tIaWO29t59G0 zckMc2}ax=a3(C%f?<5RFs;%Fr}pusWa!J~Fh(dGx6C zkwqoZY&*KU!!edV$l-6ne9G_1K@$hf+*kyKSN%X^`|^^9V)7u|LWgDq5D*FaEr696z_c08FEEH zlY2vYYfnrfz~wynV-X=%oQ+*f%a)aaJf7*U+ah;q?2`Ha1PI1_*U<;63qSVu_BQ-z z2O1mv@iHFBr3H&Q-2QHj$>rwD1^Fg5kNr=-uODCbFGLeJ>HnYMlg>?SW|Bg6IJ!h9 z9(`a9TIa)Vv%g~-wHKBsq`V;(TU$j~fe1s2HsC8lJ>3Nyf4}ueRM~XZG7xoj2{uufl zse~9sAQ9lZFx4YHvBA)b9vptmaQN_H1O)L9n)x9?%YO?mCZBC2na4szWk@y`Ir~eO zS8n8!l|9<#{ulzB+f;8J+1PjQ-K$2QfQT=&`R4WO7r{0G%i@swBH0@X>wFV^3b#ju>Pg)`FjH1BJ zmIxD%-MYG==dX~Uv`}93AH3V4oS;pY4B?ujL`lkvv;?>5fpTI;qX)8v10*dcaZ6N( z`3wwkc#R0krAqpx+*HrjZ0lMkJn($?DaXV{r_r9nkR>?I{e1fz zs>aot8eRC_AaQBgo2{+kZo=h?dTam92cnz|`! zKF9Xc>tJ1oA(-&Mq=XCJ8yy9=p#>5blM9J=0t(RDAtoG~nH+I$zvm=oQ@6PX1_p`v zN)mWTGD)NbLy zz#4x7rpb9l_I0K+xQu)68!stX`oKzQ4pR){QIf81;UO|Wat6+M13f= zVh(7j6igkMV0R?W<=BmQn;6qNI9Is0XX3MWSv0XNdh3O5|KS(qa)g> z8X11)P7vUTdOa~b@L0)Of)cOL^+fkx;Vy@+^)t;dlPHBlhgLz=1(98Zbp}GAW=NC^ zdNjHGVi}dwJI2`l1t8w5zsFz z+!nacx0~JzLs{n?<~~b-c$cPS6f;ozT_hAhae6Nj8zeuc%jlUJsO-b4f^_ntj_ ztUM;>qVG@h2SP|Ain$^em3jQl3Q5}rRth6tUSu1C_>d|FI3xyb7oPT!Hrw@onp>Rb zNnI)xOKykD;Cg%?Xfs0^-Sp2e|8!a3*d?*l0=Uu(KK5>bn{27F@zTuBpk>T6n{wPyTtfrGGwgak!F_=3ec58VWmlT1<&u|FG5=e66nFT>eS z++B)N8FcL$vr)0zYDx>18h{H10;W?ewn|D8^*3SnGmrq%yTfw#>A_o3LU$$82Vxzr zpLxDP@h0;(kh}=N8t$%N4tEhEw=9rdO;ic8uEa)0RYNp_9|;~HIc+t!W&Ngz~;_3ksglb@d+u`c!9>lZFu2m%QUax|q; z5@icgnqZMd$UprrPMb~;KQfI`7m~>Ul7non{jmHybqtFM9LOJA1W4Z$A$d08Luz!_ zf$R~i+;!9@%BBvz11b2bJjigQXRT#7NI^o+ZgS}*INSp;5$?|nglE|g7%OJ zAvJ;+f`;GaYbEETZ0bl#8mjXp%q*gnBwbU1sz4S$U788|iXTKD>ti&}d zVg3NwrfYZZq-DzAxN!s9mtQtl|2pIaxZ5wFmRp&u!ivF6DPl@+MK^#U+c?puG*~}& z=Sj6QD7li?akW~j(sQWgCxS*F!o zxbn9-cQIh@x7u@F<3if;2hB>lCRy%$sU^-H*4y^W6#ws2y|tHXYHA8Z=KZ;WvKukb zt9_3H4BmC@S1>-l&O!Un+p4|8g~Ump1L^~N6J!3v&JtRF|Ed0}fHfdWz@$|LBFp}~ zSFHpzw&W&~tN;xC%>PH*RC^(YY3Gq;7nJ_|l+~&&ygtd@ilNmsT52~Rv$kvvnBqF= z^Nf^MNaW@?!-I}9<}(6ulYYG`wP#)p9eixnI$!}B@b{%7{Ix(tL0NKBsj3ni1gJ9L zDWRkHAy@%IC}OdF+H>Rg7wC|GV2lgucvY9CFG>Ulgyhn@r5xd2!3lb2m3Tk`~pPh1NS3WwQ7ovo$tMJnq#PI_gnfE{Z*M6kC9IZ`0xLD<)8ml`>(=nRhk-Hs~s0cd(OQH z3ZPIo@YQ5K7F0cT=W&htw?+M#J}@9^?Gq#8dS%Eoc*RNgD~@4VwI|(g-~NBJUjM8b zmj6B*|DUJOb$8C2Lb_g6szN_g{c zSMJbVKt&2jNUQ)%0#x^gWrv1fz2Ph9-H#tXWE?!ucZky^>{5Fl{PG*nH%Sj7VgW@W zq(FprkjxyQbd;iM#y))a{^DQYljAEJV=e9pgk_X}hPVvR@$(P@`~lCo=oywI+kYmp zHH^2 zJ4<*c=@vu`Ir=rSw+kyxeU9jhXqIT`53H`&c42{Ii-L(v}vFhjMVqQGWHOiw%#s)|L?HgpJ9e<;*M7~Z++IY#^ z8*>c_neX9_lStT(ojaeyPzMFk{p0~D12L@y2!GQ&ra{bfo0gL$J3RO4=~#YL-<;=k zp8f_-)LCvok(GkvEa78+m#KUUTRohIU5=F$qz{KlS<)I^Y?F0 z`Rvr=zOA%TNGj>Q@`&s7Y)Ac2Z%&|2XIq-pzf~Q<+ub>S1+MT_Wzw!27o0} zhgU_;eUiz6dCZ4_?uSoCqYz>K{5OA@#72N&}-MP^A;JmBN~Mq zhOVwI5BfIv4D|tyKjo#9@)F1isK5{6SPL`~5+VnN5j*jg!)h~@cBRNq@tJRFXk3w09<(CLKV}**7{$-SLt%rK zVepxvfNH=d6%f;5S+|b3A7p^Ct*!0%xaVerObQR5AYwplG5b~})HWWB(;{yOEK~-o zvmaK+UUx3sXu?_3V(dz0wqZUh04fWVVrJ}rHDo&AP*G&U{y8+R0gObDnNDISAtGJV zL_~iu&(N?iUXU2+5+Z7^lmAl)isF3M(D6iWO(^S`L%k^lVuPqrtle;dhOY^!Njca8 zVFV})QZ|R~HB92Y2@y|DPR78`z*nc{%d55+20V4@W8(GE5(+B?)H{V!78=Ttb0Kq( zRi`dA6}pg{Qj)}%r#Go~Iu9#c7Vg(=i@JmFd(!w4ZWGSK=lBBFvBZ#|ie0)4o9vGn| zb5bPtmLjuxvj)YCho=}&G1UI;TMR87KLkXAug${EElLF-rjs&_Usgnk##18$O@VN?sfO+Jbr$zI{sFD&bhWQxRQaHRya0_sSvj5Sc}jnSdDJip4;jNC zLnM&pGtJAFAI5 z>RNdCeH}7t*0y%mhnXZp)wyA>--6wZ5ki~bEJ(wQaCR|_PLA4O;aA}SsDPfJgS6kh zcIm60ni6$vmIwZpp~4;XsVw_%ODsUFX&eoioXUey9!%M*HESJegLaE(rLwd0_~PKI zni^Js6SNcttqJKS%vEalIg*v-znpts^4)X5^1L-IfFoO~gS@e4S zK7+0_$gY{$*}oIHSi`Qp$l?W&Og&&<{)I6SuZqt`Xu_}GzdwEE6qM%?`T z>yJ-%XCz9)c_kyj2~{|M2G23#Mz|hP2)Vg2BzIcUx#tg5P56Av%UX#HtD^=Ig$y0H zvkE&ca+f5f0E?d_`aNfdqKRCxj+2x0HP^f$_|h*h9{op<@W%`g0GEFI%)7U5Gf#h& zM%;NFrb2O21b=~3-&LrmHM_#wvmAR3(YfMrYv51dl%tHHV&FG#>{%j6;|{@@+JnaW8;DtvqhsLU`U3h~2@Xc= zGhX6I8pct{+&8iWfrugP>vr#j#hiK*aSixn>TN)aukfVOiXwLcOkq-l8PX*#3LV+d zJB03UK@hlo@`~rq?Ke{OPelqR{QNbi?9%^@j^JZtrBKr3_1BW)nqI&5!o#nyIXr?V zdJzi>VIwl9Q&p>SjzrL4HId9ef}kpP@<^*8;)Oe7p7Sz9Ck5Gy-@YJfF$p%H7a;u< z-UWGK4?EHTQv8A~ty3_R@);z1-CeGgV3;riNvumQv35k^M?tfmIv%MqW)3(ml~8?5rdz zj@^Qyf9B%pKlE3}<}O{L%FIeDiT7T7shk;hd^-AWax5vLO~gpA61DtSyh;0}*}PO1 zFwSG}eS&F+4j=w~Mvv!Aw~3Y@D^{mReD=OnGP60j^o%H(;)cg$ax(hQ+EaRWc~P>; zc_`YLqu!=|1gSo7-uYN%1THITYG!CRxQ8Ke*@DqhPwRUa9Rj#$TcE7li9j{*n9{>G zL|#-43_dOXM<+?P>XdY%bRQ$vz}Z(gv<0q89D|~l&Ya&q`Hfv z+6}%a<{w_b#)#1pOf9;^j>9NOP6bnnu*%5rNX`yesJ@6dm?9!Tgh*l`g8QXrlROC; z7;)SH-qJpfLl35W70J}!bhpV|Df(FAqvT{kVtok(B|L-`g!5!lDsoXJ%y1G1VGg(? zOmHUA{DELGB}O_u8KoyhB`D}-#YT-=UVz%2LxefFPoD1_LZSj!s)bSyuUSXh50-$= zJyZ(yMjO`!cAd1uPdKm#^KSESt_m5v@tmDVQor}uvdRpFp0K}q6VG6zo=Bi4hwCZO zs=9mKFcU=3`K(L}K(Kt#|MgOzqQx6F{#C+VSBv4M64FYh&ybhJsWvR2IcPXc6d;wi zzyCteNoK%TI18pQu09nwq-B?Y>EciuD5Qk&fu!Oe4h{~Af>%N;R_rtcb(df`3c>4R zIjg0}2`O0c#AqYhC^;tq2!M_vlbfXL1AP+@z2nQ1t$_GrHHUXn%$%G{kw7IU>)^WJ zBl#k)0;Eh)(2-wa)Cfm$sZr0+`bc(ndMJ7>^icourAyJTn8{g5sFi#>cdiCXKe$&5 z+kPpW!DnP<0%uVajyBl997xtJwy75Z^W7JHP)A4ih7GA#Mk4-@ZNGf|nrMC!8>4mr z>0~1jRRmU1wG3?^5IRF*keK8mAQ$ea?%p+t_wSn_OhHbBLA2oaQB)udICCm>Wc;v! zfdaS&b_r(OoN@*cs{;2RCnz8)a-$XVI~RI4`u-8TIxKi<7^&*Jyi(b7_*wxlt@cJE zQSXcNH-VTiz~DIJY!ZIp!USpJ!Xctw+t5{?5pD43_}-jeq~KX$sE5l5hpD%G`^JMq z%`$ZQxaY?G&zwIGAB?TG0TK->jz-b`ybTAVWZ~qbq^lXIyh08_LP9=>1dLaC&hp#h zM1@N*V#9WLOy}doH$>4{bPnVAG_banWFkEmHvl4xC}HkC%zOyGgLIPz99A5ro;Z9C z^2u z;U3%ILO`IwxigFunT0AYmRluaBv=clQyBoxhncoHHVH{PA7p=r4-4FU4!qjXZAyt@ z@aLqJUi1&B$P2-q$PiFNIPxN|4&MY2-&D78Y^7pyPd!*Xft!#3Na~7My(!QJYy`J# zc?D%u*Eog1N_>!vj0^$}Y{+mwFE3Y3oGVy&r;I40otD7xRM_GGn7-d}YLyQTY0zy# zeg#7|h5a0eWRRCJgNB0CBb>>S>Q?vh;~sDVDh3uCuqHC#*DEmBjT5NmC+aM+we7Xc zHGwkTyB3RfShp@973T~Z+a7Uo*@@+H%+xe?MLDVryb^sIxGTPt&`NR5U(lEOV)0aj z9CEU=d3=($MWt>dji2-0+)-D!?bjQy#+C3mbt|HkWdd`*e6diLZ-v?d2XF6QHg;9v2=_HgT|^4J+WKw9rGFl+A-#l zvu4cc%oznFuHKFIeGgM<;J!jzODZZVuHCpn6SS@0 z@C!?rkr#oXZk5iPe{Od2CVQ4a6&TqiEy&MNUf}Y4$Q!9EOiWGjV~ih6_2vm09Me4( zEQ6M-5eL8tje*#7RPr=KKIv}X%R@5GttCg!f!U+I(NWlRM5Hh?Gqe7xiu}brr>5Tt z2HXH(*8MryGsH!rU}yxP%JK>dGV-fjc?N()oU^;K7gi})44wjwhSSf0aGPD#Qp#~yu6PKN>8)!N)*d{b0H~x(ta}r>0T=;#=pIhDg{uj~cs7h4p zF7A-=FUfd3^k|fv_eV}*Kq;=Rvq`-HmlYZ#RFSNP0&NFE0r5$gFrIiax=_ec45x$< z$c|Xe(R=KZ>1(3lgJa%Y3{1@jDw5yqr!gB@TT3g@l_*~Z*H!CHikGt{dm=wxO-4Tc~DZZpSe3t+zM3P8A56iy{Mepqo z(pvD+Ub=%`J$P~>?f6|fW!+u>_WJ<6e_wF|ujc>jL(&LdW%B0`5-r=Op!rZR);{E>X-rRr1k|uaYbtdGjgff&P$h0!5i8z>oJ)l~ zGLG>GLIou8Vj{AG`>fp{hqF1zJR9l!6{A2{I9%$9_T)(oV`F23LT7Q<&XK!VbFy)c z7t|m#M506!y0dFZP8ilIA;M075`b&GgRZTt7LJ9_uM@Wy#S_8?4sqo|`gcrmG9)mt6!v)B zpgadcyXAmosgGD zx-MGo=%mc)FFOQTnNTiTF>vUk80)1Ni-@_O?nb1%A>{+j@({QRAvX8G+|UX{-=Qqw z^cn_|jJb=mF+FG9Qe+@@z)%3p^1uaVPS?*BOD4(P8`FY>fT}Oir2#ITg;z z)fS}wP=;|ZI0rdI!Qki2L2X25aSGz;V^5^n-fA*Y~H1f}6{ z(KTdb9%)gMp~Wg!?=3+Q0(X^fQgY^e?t5T>d7pW;yXn?KB;PsDt~<6 zy}fXEE|bHRuE(t;h+Dm4otHN&Zf2(zydx5F1h zfjb?hOOPPl$wNd&B!i;Ry29n=+U4h15R^(^WtVp7;Y*N+G}~sJg$0q3 z6<{tbmnbXHV`PUoh(Hc0!5)!BYYX8+511hj#U&&tHRPUcQzs{2;jArat7Q0;Wb=g3 zkyldWq-0zG!1e*4(d2BL%_GQJ4#rBSK+czIgWB%KqF)r7f45wvl39v zH!p8&Yxx(0@iA_e6k)mOfExBbGK$n5u@$j^Fmj~e|5x0Zfc2QSeg7&;8A~;mA)}Br z`x2v}ND{`X$E+jqW5^87yMdH?v>ie3hx$^|LrF<|E>&TFRNKf$^7{+g;Nk9eMH&?Mq$-AvAfgl$$RI}38vI(0hdE%((lX$J?P z|CsfwR*B9<6iQSn7wG#)b-IikxM@^rFiMQon*O$l;T+4#og))}eEG8EZ9-p^;n_~7 zL7T;j8B{XPv|p?Nxg465qwD3Izj^s`b+VpoP<`j!D0LV%s$qi$=PzFlv^_Io#0a>U z#Lor!`GN1-x*hSf47J|)2&^_pLj-+$cBmgec5EQstOR!82(&|POvqBlvf}h|qhnnv zPFUxaoR$e4h9FxzU;eGJ4cxZHs8NcmYpG%D{yCXEJM!R4zOMv_Al**vD8i|)@Nx3V zSkefG4jbl8x;rgw@>sxPO%US5tjybwe*dBcKFj|xwb;Q)9|2M>G6;VEPv}F+z@e;>X>)OI&7w+HUV`u zF)@*yR8SI7RwPl|a*~#~?6-!`C+^F>R^`VG+w%5%P4Bo4Br#B`Ltp^fdxJe=#cz!N z9uUWG=p)LS3xLk7u!1+i;JpOd>AMm5r6^$ISLmiRQO1FASA;5(?kNEy3NhpY6N>}k zVa$p#_Fbrtcq59GJ`$pVEoE04pzLKHyP(EGj-y~75NZu#U5#SX)$?fDg9oyE>DOwe z2s45Lp76yst*3zDYnOCd@E2%$+&C;x(k!SbV7p|)X^xJtUB?b7hz|9zzT@KDl)8xA z!Ln79!{blXBcyBBH8gXw+z|c^*Oo*j5ho0D5+ODF`w+il$Bv1@qSHpukqtccbUXnm z-G&2*@k^4=Rf*=p>{K^-OsYlou$tZhj%|vgqbHzsH6+SwgMwxR6f7*tpF{Z@UQ&yy zUQ-p=VSs?w1bZL&IVJ?6eKTI~q5g9pMAt=GDH?HMxTa4RaHsv$0BjC`8I;oy>y1R( z20U@CWiGixdlQDYI=5v z)mV+eHYGzpdEA*Lomh7@vD=OvKR$q}L*Ma-AAhu?2$$m=?+jg>^~_(QFt&ZNiK z!}(rerufu51S6INX3Ik9SUIP~6C4ytsWFAW#m}>jm}O2=?Vc;;S@R#``%}`=Ra@Iw z7v!whUT11*@84~|26Gr>ao-{I*5}HF9W7m&A#J*s4PFbEGxiVaW1a^GVo9`={B=o1VY z(FOSLQuV8lM+9EwaoAtfx`SJMx>rSyapX(+pLByhW-a<>DRFK#`^IAX6qahmmw8=c zEWd|X_{r4UY}(hq^|Ws7FSsEInmw2PVCuR0YxiKV7yybEZ-H5ZIn}>58%d34nR>U$ zIFX^1FzjGXVMX)6+bt-#LXoGuOTWE$X#Ed*Qi)Z44IdhOs#^1wVH(4Wbd`#GuP-VW z1TTKJ@hCyWxj>9|y5@mh^&kZ*zwBR8@Lt~5dKi(|i4PBI_Ula>RrybW>mm9iqj+B3go=3`ue}w;{Er3g>V&>7a3qgoQ{S|Ua4aQsd=L9+Vk{ZLAPnU zxNfVWqnE6C(59t-QD&XWUekUFQuvn-PIMu-=25%XWYHWXhesfguG9a}ME`P-1BKwY zDKQUD6Otxb#XQ+&8$CT<+FY(`Y7N4vzFYA8u_Jmu{stm=d-@YrACuNj zU;iSN`T=YGru~fAM`~1E$fUV|Ujm7X3VL|jNUB54#*HO-43KG1ngp8B7KtPnX8`)* zt=Gg)Mk{fDmY<5kQ&WAN$tsAQ$zh8a??GKNFennHa0)66{RjU}n6$VVmnOVKm&scI z*)PzM&OcY!Mp8WuJv*?+!t(z>rt@Rx+)lEQmX#DbY3nNOe1;w#dsLOCQf5IwC9tgq zzNd3|Dvn#ajtn#lpvPmHj>pP?hF|jdX&ulRhqcPh>3$BCH(5OAA#=p0N5AkUJvP#E z7X5=1Ghm=y5zBLSrDiwQ@^;S9u6&-7<^qRn;25DbvxcGHi zLkd#yAW%|ROqx{B&CQK-1N@$@vZySu{5%}&HRxh69KpBMKnSk5BiR-#igV?Pn4(x*QziEu5tQWLDCd#NEQFZt zIPr7?7)!BfQI^SE4aG^@k3Lu|?vSzv4-gkYET+YB^~x&v}02N zu}N6uID6kGQoW_Kch$q{KNC$1{ne7@fPPntgx-aUT?3zIhq(ahP$RhJdI~n^Y9iu5 zq&JXIwf-{_+q7(H(ptso&BH#*d79Audo-Q*(z1#i8N2Od(>VyYd(C?&t;@+XXGAwa ze!crIzg+L}AO_u_aL63hsv$qAC(m$E7LYfFhJrg)O`)C)ixN0ZoEGw!3Qgf>@Y6ICP-gi3uMqG@V3hEs5(%LQL>6rTD;yMBS zlqo{>Kvlsq3Z-cLp|O_52PuW*Ytl(-FBt$KS(D&{cc+))pOOoVj}~=^%rJ+3?QO6~0bCPv{i1n_8Ib`h>Q3dO-#i$^6c69 zYuCa?WQu;9cPDcph=ytai2QFLB&|>7MkX5J(!i~9_-?nB{x9OIwy2l| z86_$(ZmdOr z&E5qRR64;mC3@`gN!yP$YS>UEWWuFku<}DY_Zt@*HezDj2~7AUfRk6)K=LU5#7_?|ZvquWAfUQJ z9ckI*x|*(wGHX_rVKcp!_R{ajS3+%q$_9(J*C}lLq)CE~qbC?MY<&`m5;|1p-45Y8|L3Q{R&Di<9oj#Lk01t5 z5_+ix<{@zjQLs3km4Dc>A^nTQZr|YNJkdPUeXZ z9HWHGm>EPtk}5Sm(r%FHaIdOr&3Oj%hcW>IOO;;MtX&6CGfC^gPeJDdO}A>u+ZXk# zNNE_FLx?iRr(%6;LfXlNdc0W_|PYTMi;wnRg-yu`a z@b<&ymd?(Gh`4hL((eC~e>|@W|Httpabed?vMB&Aa-s^hMd}gL6DG`PcBn0kHGWZ% z{@rev$@4|-qGU@0%tyx=iu2~@pSu(6U3=KD>sdL2t56?lpc6RK~p7Xs%+y;U9_Nt$1(nknp_ zNIG`+oMX9Y1E7w*$Co=k&t_vV(uZbrUvu9q@bEAn{DskU8ve`lgDuYi&H{k|pCjq-#}s zQ4rU}!qn7S_sc%ED15J0!=_D{jPdy46HP7ey-!;{``dlH{lVoL0x3_=;Z`)A<;GR- z=Fjp!uc(cG+*8E$9Lb^q<|%Td?K~Xt_*Hzpd)7dZIJ##lxU1Xavm?`rVhmR-ZLM zOUv7^s%bM{Q~A$&U;epH#Xq0+uGPYHk($LI%$rJ5yH+hN>$@0jZIf9r(OCR5Xl+=O z`H6QD6HNvUy0Yl?^{o2dMNdn`3iRb}c-OSj_qo|jj9Q4Q60WbhjlByEZ%Am*{KAX> z{{3>}|HnMkB>PI+%-lC!rG9wXe!7Y0xw|9l$6|8QR`b&cyt>S_cO5TXi`R(C-j)4H zsQJFACj&1-`mhPt2J^MKKNFgD{P+hoPx$`&>t-@+Y_H=Msx@)qqne-Y-do976)CRR zoG(#?dN2v3=TdX7dj7V`+GWxlY{l_ZeiDS|xDT!sBBJTTCH#LZObrF{X%NUOHI1H6 z+g7a(fLYaj6Wr|zYJTt$8EpfMlHA#dz16u-UcLO)UR+h1UIuW z4tXvysqj4{i-@UkR6G77)@exS2moOthL?~FG8Ov^NqV|ro-=T~i2tM_J&5{=7#u11 z+^8b|IJ@5N|yIrzBXf=6RQigFeW)^eX4#$rKiNTI-Kv z^nEH)rNQXPKV&=fgwr4j)GoDq~pi!%VHDOuB|Rt2CWlUsz1UHnXRMr zhkT?J5=jfYn_ksFCL!kGVxbKLL{?R^+OwNML`8!ZynAkQ%h4 z>wo{fyL?u_!~QV;Dhia~U@SYb|5gQG;6|x{EYZrZpEQ$4|K_l-)Kyi9QTOhh%oBL& zyU}`z)=Xj|zM%!98@%VCHQM&{;!q-LLXLLq(Zfp7#IPG?HJ*WqzeoTrw}$A3bH#6d zBvPjKLZtDeI}8o7q>9~y>C3#B@c2zsf+c$#Z}00pl1DNm?ITODHm^^*VQa_Q>gt++ z8>lAAKHDMrYD;}-6XD4-v5EG=gT_kOFc7mr7{Hb0^!$?@eu9@Zfry+yq_ypRYDx06 zJz1LG(Ge3)S+Zz``lYWdc4XtQcXWfRFrm_Zc+PcvnyBBTJ*eGcEfCLo)EO1yXs(Tk zux*X9ah+u-c@uIOL~e&_h%rABD5q$01aCM#k_B-EG}eAIxIF-DZSt}^J0e)A4{C}F z<#ftDzo3Jk)c(r919!694h`RX!SXqEfNV{Y0R(y+duQx_$Ncd~onQ1H%^$1s=&~?> zgneJXa$3`v@M5Ir0ia3-(RV?@XFE8^j1uIT+)AaQFH1@~o>I0V|7DAtDbAMv#ELN; zQ2dQ;M(*EC#=}%*=fTRY2aX^5BX$v*Y;Pmf*UTs=QxtB+MD4fR?HH)30wFtvIm625 zc;hmaX1>=ZM-E3d$^x{{k`No2Aj&{13^VhyOJKVHx%VtMnUws?uL+by*7YkP-u&tS zFzj?VAYQNgG|^}C@v+eLaLNU%;>d`b zw1w!hLrNA_dOd;+_VV!OiDs<4G`4($1X*(=UqWXUoCs5u!tfUG_aSzvDJeUrx&FS= z#y8Bh%qXR;*G(3Xj6}mrkO^Y6fy=qrw;>)v1y828y&`(Lqw&|j%4RA99z8tjib?41@p+|d!w zLPM}3NrlOwZT&PDA>`(T4+> zumGQNR<}9vs>!gI!a^cIWNzyfc&ekz@}|k41cJeYts)>H=i$tWrTvDC9rJ4+&~ndp zv^v(A?q`a>(6FXcP!m2<O+VA{jmad*MqvdR}F`9iCU2z`SGn zDfWYFCzLl2PK-2+XIFzeo)nd;fjl$lXlY@Vp9Gp6*IXf!N=X|5Bs&N@Ld7A6HUQzo ztSc9MhPLL^Clmvs>xDJ$pavT-R8XUcmO4LwvEp+qoh#isMwJpQzx&q;Y-gu6%mcs? zi~emDTX%*GkX`;LtdA4Zi6E>#!~1W#9Q>iPpjSu4*v2WQ5C);9|4tth8E<8E>fc>V(3Lb2AWzF66P`54V+)II4_}Ioy zEndCI2M(9V9MLEa8n+|8&<=4+RbcJrJb6_Vh7W^?2(F1l9cS4GgifD1Q|1K#QXcNr zBb22`l_J^z&eDTH1w677nMYd{GA#B}aq{JEabq4s*ze1nT_44)_>MZ2Cf$sK)#305 zGLl89N8f^($mPh&u%7v48GrM5RFYI*N_T@Kn&jb^syFa{QPpU?r#zNAIS5)C_U+`U zOUZ?> zTT$kXjA)930?t0`jRR#9YINr}rG78KKSdSQy+@C3oTq}T4=I{6lrz2u731TFQ|Si{ zYFBkC=FOOU^=wbAy}|8;yjZ%c$AvFifJ!2xTiE&!@87GAI(6scG3vBa16*6TY2%M! zp{{*EFJkT_zlK)7E>ujL?10cm9LIXGa|mz=>0Bw8tdZ`P@z zA6`qlE{ZU6Yvx$op|nHhCBYOxO)Bjhd?Q(l`me>8j0aCf%(KMFVj30vng8pKLRv#M zLY{Y-l*Ha#bBk@V&7Bg6v+i}5bft+Y0B&7IdI4-J7(cv7r$f&Q0S zw+|pCx{0v_NJ%X0*ROBC#(=FY#G+^5gp=(RGlyhmg_h7NfuD?p;N~8+ZRUq?&1WUG zq6f#h%r^r7zQ=?e-+1<;z&90by8oEHs9a!YBuh%%^&MhimTBNNLI{axRVD_BwHbA5 z*ee1Hrzq!<_c2mIMrVYIiBE!6d79K~*Iom@#o*Af!A-_sp;*rgzqsXFF#w1HpVGuJ zY0qf>2meJpX$)J)wy=L*#e2w5ro$6XN$9uxM z*;y!!6n8{3B3P!>0jY06#J9`^Xdn0N*>r+xo5)B@8W>(+{nE$F2OFB92%|8^h=br~ z)nXTk7Y@CRDoOHQrL$u!$~wMN4KuxywjN;fumS&Ge2x=m3I*M!!bYOyffFY6$6EyMsq{y*mf~QHtKcZyPPtdi zk6z(nY31?dhVFk%9W`K7|5hp+&W3vXyRV)&YJjT2hDDd}BqipDwVrcuT9Xi;jbmOL z?l|a*K~(C>xy|SWY33&qye(=+Z<7 zXIntir>Ce#*-ni72y$^5Xui2?Br}_+qP9mvfrzz)KELDI5TjoBU8d1{(`Y2wM@b4k zr?zENcv^&&G6;v3Lt;Edz=}f0Crvzko~1Ex0!ZD)I*{@5rhy3H5CTnl@TpbZ7^7uw zE$z=ObIs}P+V|YA70Ta`dOG_bU;?cNh#&glw7!oFCP`e18XUti8&l!NT*6?byBA6j zu~?8rZ(x=1!TEJFt6~Gm0U%010DHXn+7IX1*fbQzs`ap)=GUP<(DtZtggAI=I&0fU zX>%Tp3bn2vgeIt~P&*W4fgqsD;ADhXQ$ai=(SwR?Cse1>5{l=y7tf}(enWbH#oDjF z#*0H_e4#yxNbS>|N_cipZP^&aY{jvV@#Kk0wQ5GcAyg<@6`MeX<&h&CW8y5#_N`av zm;@vw?XLb1VVSn)i;cN8fzi1)@|XxGDb8&Le^Aatw}I=J?dw)vth;~3n?5c@*}**W zYDVQHA8MWu&V(*B!nJmiYkJ=yq^Bz9=?f*j;5J;7WKo4qLZ4M(i$I@8E$I$LKDsH- zj;$l&(m633-+Db5mA>dQ?orvcO%9%?c6)-AmP#x5!2%AgnI3u-f^Y3r-WCOZHc_qD zi+X3BjhcWa#P`8myuYJI%cH+VX z&uI>!N4LG}n$G5YF{MzZCLdUa$-dy;<*B&9kfd(my1x!vy2O$(&-|9hH=hn1RKUph(XIn3vUd%L3N^Dne)Fa~>%2 z{+<3lcvB>!9IJ+vLu-$TOz0!b3|y0PfQb~W=v6T>uAa`s2}(X5$ZXETC8UPR0YjES zfAv*lo1Le4gl}@1muW#MDoq~?H-pQogwME=-^uU^Z}zm z$^vXNXx_Ze?m4aQ3qV9RbmTISHaLbgKp5KF|KKFmhV@MR1dk8A1wde*dtHTfSpP$S7Au(yp;)^3aP87e;HW9tg1N=+_#977$ zBm=xzI-+m<4EK^D@<^cr(_sP^NwO|`Z(io~-OQ?$K$dIQh7SD}(cG}K z7j!maHX=X3p?adgS&Z$hOjOXu%yx-V)*(MnaHD_(VTgbaqR!d;7VRWVnVOo~*m2{g zQjmc+cA(AuVEv$QFg9e#X2f?QL#ri8k}S@+*}vgt5t9$k>8c5xQ}rLjtmutnn(@FI zbkwL1rm&g{hCWqE&PqcfF2N`GWroiGSk0Zy9)4N|TH?^VgtSqg{IEXa(jOHXzx;oBYs zK?$?v-+(me140wp#>)_RPX8ybULD@P@x=8C|CA~^gowduIhhxXM240|jq-=qXs=t4 z3jJryXidSOYG;>O^Tho-xdE*wPBwKjIBQg}&q!x7SbfN#jM7#4O>Vzk|D;UMxhH>< z|6SzB(nDVHySy?4N|!J15nsp{?w8?T{2tkVk@EyvRS_Qlvma<;;8Y>=@g(Qbz)787 zWSX!3eHEor@oTjNu?hTct>*A%sussCd9PwwPFPSy%J57758sU0@Ma3HVlj{%Mc%;8 zMv)pVTiSP?Zla!$G@vg&Bt^LigJIuhnOk$$xDWe0_J+ruHs%C)f~*aHPbYRVbq9ys zICTw<5Ro`DiT+;W+kHROVbxhh_QLO-j0qGGTNRs z=_?C1wE)W$?GC%4(zNM5=ZIl$6HAH*#AavaJy)+(HzjTi3)7dUf;MO%{5N80^etK(Ia20ZHua~RZS6`n98cd!s95#fzpRKuob_NK2{KYh6-&&;*uIJz3ISEYsl+q>>E)j zeHaSLSo^B6yr@i1oB2`)V`OYZz*v8T#+;)2?|tIy=_Cd6!&JiMUZbHgA@&QqfoA zY8NL9vJVHpQXpF~^Agg)J}eGNwpwXWm%TZaC9;Kx`1)?euX{_4e!JhN36GYx)XMH> zRHaMwJc5c|rWA7{BpQ}6a&%Q9*p(Rp;3Z9i{75tsB|58+Qe5pF)JaRG7EW>$nGCIo zk#c~JPKqAN)-@nxsdHSxS;4mDrzAhaqHK!lM@IS9uUXSer`m6`6I#mcS1KmmqSA`v z70do8qCqhmJ9c-^ZbcM>dBoDR#9t~oY8@iau10{TOWAqbsJP==O*ze39CZ|4JRXF* zV#Rt|l$H(}@24uA+56kT6?9T*dDz$)_7O4ECIEUD2L^(vWAha{9i1sB>#5D56#a z^ivVC5w4;!N3>^$+FS{1JG=`crI2wTO79XyQ+d!b)V|dL!O0yu%5@@CY>jC3P1NCO z#v8scHPv{hRROECv|r4U!jTg1u2L;<23V6g^U;o^%q~SMB{|{X1Xo0T@7>qS@`347 zx9>e?Oq;eE;?L}tgnz9)Z5)2ph=+O5DK09I1R&Ar5FHZaNB;j*_6y4xj_^6AW1-1- z^ntsBlJQr@jBd&Uwv0l}A%BjYos~mJ9)UqvhzL_&i}LqCSlBltJ-pw(q_$S+l`n@% zHYV`e(xJP4XVO1n^sri6)z~n|;8Dm(u3vwi z1==$eqmbn|+7?A&ikpm&-G46_Wtgf=REQ3F(QmX}7ZlwO@ z=$Vu3`tFZ?LxV)3MjYNMk+V?bvWDAr=`t97*P6D|#Op}+FM%6KmUL)0*8~LEc@DXU zndjZL4v`Xw2>xI@gY7Ghx!TgJAm}M!yA)?V3ZT)EV9Y}|!3sIosoh4e{E4OWo z?>Dxj?TW5}$C{u8xmo?KM&RvbgrqJ?9{VX+yw7M0WY(n2CW}*#m;|N6dUWS;GWval zg@u+#IQV}q{t=Q_?stxRfT%p=@L^NwVCK$!INNboMfzL~+Q5;e=6+@Y15hlXsP+KQ zOizf(xEU+CEPo}$o#do>ntcfVG;s_<1hkc=Ha0#Ziu*umh>H2H zkw-}37hl5S7o0&Dy2P?83=akUTiL6y=y`9e9<*-x0crzm*}N7(Cb6M(n&M z6PO62dn%*Nfj{w$ifMnmemMdLY7a4|2-bxmg#UP{mN_L0 z#}e9d{tP&2ULe3D2!>3Sm7WLz0}_+ShL_gXs=wgL#TwbgbX>xXW)}lOQ0>UTUIqaS zEoQFZu}Kexnsy?RfJ%?*ygnxr$fgAN2-h)pZXNo?_NS!X=O^qVj*$un;oB6+zT%EP zLV_2mH>IdyC9z=#_8u4%7%0TL#HFzhih2W*S%v^9_%2it#BLe?fs_%&txab)vj!0uEPSQGMwR#A^C6~ngpGafVZ0`}rf)C6^4 zJuf1Wcz3aO7qK3mtgx1%kTqGdBs#wz?;EYij&mf6jYxY#yxEZAgWQdRv|!VuiyLQF zcb`48g_cE+3&qv4d+blAqcO-YqxEkkSq0fEIKB{O?fd-nmo8A=frkz;_Nxu(=Q>Rj zYa5$dQ0+(O+-(EQW*=n>4eBBOU{CA??8$>%T+U(|h2o#UCO;G#(a_3RM?#r8?BuNC zr{jYk?+o@KOhxekZ`Yn7Ln)}7>vN>pHUdKc0{2a0r#->0drm%!4G|C?kh}2ol0_rO zgnpnQn$T(NX11?I3R($?Cd8Aow3c25ZONhIuUPA6WHv^AjQDMN2zH%w0~eqQ3N;%K zCOwVn#3`+pbN8~+1L^sBpS2AJ+^RI+=l^OOA@N{9Ymyh!={pni`ad(1;{wq)pjNVS zkm}1+8vNo)f0UAff1rXV^=g*Z@Oh8B=o-1a)1 z+VK~$&O4x%NeoTCw}gTte#Ek}p7lM;k9tXBTl}G3ou;POPw?}q#Hl<&67D~EAm%r` z9El!>(%@Cl+dc)Hmh>$UqoCiRg{dwX90Oo)&~8622Ix}%0gin=)jsq~PB{?}phppV zr~YzWRe-L%dPOW@ z!vNUg(TVJ`;VDZ{k?O3d6BZVBV05_H^cYWU^t+B$)K6b}e9sZ&5Hq=tt6oc#@&^nr zNP>~5!3r>)W`O15IX(#L#D-zHhc^m&Q;~c8;)QC+h(oXNSUV)I15Ie2lqC{|R+d!cLrG$#VUwDKd;eG@}m3 z?}9&p`^c(|YU9yUyDnD!x(AAGI_<#@gVaT)H_12J&%?G+m1@$wEjJ_M&6_REbG`^zOVJ_U<7xU6T_tW-@e<9xxuLp3 zT_GxFDI90bQUe|$bNTdRujLrUrb(PjkC}C_(+BmAEp38rL$(D(&;#3SZ-HA;kH5NewME=obvKh%2YbD9c?loTf=awGHHPR<(R;d&+T0MJpU zA#r9nrHyErsL?%Pv(6o#tJ9UwH@a#iU;cA*Oeqb;y@Bx;vX^-YudSjERzPH1gj1*e zuDdDt6zYG%=(OeH5V`cW02*@v`X>sTOXm-~XuO8-U&I;`V^A<}1>2qAA`}E-pTMxe zl!V{SVrVU7!U4C;;6+r+M;YlFMZx zRKJ#SSBmRfe7KTpB}k5eeU8HZC0`*c;SKg0R`+*)0=#3d^%Y|3@R*TM+JjMgk`fGOJE?1u#ku*m1@n?1f%JiM`uB{pWJdAJw;`nfgUAj!_ps#T}XWieR3 zOikbLiBFNAfm3ql5YK(CZe6wT`7nF` zkk*@WXd2S&;mtOU$gAof0Z@&{p`8=g5IW0T*Y~wDsxUD5r<;>!T)wOOCqNl!=azi= zZZ%fW%m&XKLN>m%_}^VjszQrj&R4CkB8l&%90zmSaPZ(%s||OjR8*zVu?hi!pWAF% zyLbS1;O{>a04EM%M6;zf$+aN)P`%&_e`T>;YU@;73cYb;TqslM#sD*BKbOxd!UJLTNCs@n4O2oUnczUpp;Xf14j zLfc|^Ri$gtz2e&w+tk1L(syOi=e+}3+JAr6c&kzP+d`7bJyEQ@><*r33iU^eFgK^` z{%_pMzcrYA|EIRszi0uf3i7J!4)%&{(#k(h3Z`T0 z_otsV^4J@>bERu^{^yPPO)8=5)R0ijy)FF@S&bXFkvbsNHiZqFPVL+US1jR~R5S)V zV=36WcD!M%jwnGAy=|{=XDarI*wVVF$3V@>1fzl(;tZVQ}oL zL3OJsxUjSuHmrgB6S&BZ>`nS`x2L@%=|_+!dNLXHcjMPS$IP~MCL4$bN)p&Om&kve*ZM(~HxVHc7R@ zY)S8_-a-bKlgB5cs$?Lt-$^%0L&+TyPni4^H9PXC&aFQ|zb*iE6P>*Hw5VVN+Q*X_ z>M0@~F+GBTXD}|2=~j|4V!3?cQ);hE2M2BtR#_+=tf_-$&U{fn6I#Q++;6s-;xV() zw^|1EWutU>S@>voYCPi9PG~P4wR0>L@$Fh)-$#1aNt~*bUA&E^Ua77COpFb9MZ-P6WbG&1R4#N~pZeO@X>=B*~*1SKmRhjk|5U`HTg4U6)x!cdr z%;_%w+81_d*Up^AA|jfYZavCJhkh$j-G4{R;U5+@)V3EQZE^rV$H#p-5E)zgdcN(0 znWXuU@uhR>7C`+Ko?BX3kmcH>c`ju7Rj3OZNd_JSPRcuV=FBgU+kLw|Hnz`>jkR?y zF)iAh8ocsueCBn+9gwfp*Vtb$Umsn747Y&sjcMaKaEu69T%(U2+iuoblK5y5FsRwi zufF5@a}MTr@82id&q97=kY)s|BLUb9!$i-a^hXFnW;#kcdi-~PU>1J=8*Wf!da7*c zi6KGkX~`ku$EO!~mV2SL3v|5bh2&0H>-<-WWawQG7$j zZp@C7!DG(xSu{OF8%sPB8W!vYG>(y0B|M?kE9p4JoqAkED?|~@&Kb4%>61PYxOR8E zY~QhCb%hWThknuD;d23JJZJVJQX7MvNSb8;833T|bz}^75X;#|7uHZylLU2Lmv<^+ z@`W^{q*Oz0Ymk$&^N+CkC2rFx!AgrWjv?;MSDyl&Ck<+f_x0y@X7@Bm3v*0YlIOQy zyl`PcnlTT*q?+w-tn02#;NkAw=B01>u)N3fpaK~fI}vV(Et|5V`Y7MAHQeb#hvXWU zoH~-%33ohKRQyN6!O`+f6>MOom8i5PH>)UpZmyl3G5(#lJ<{^VsN3|mSGiZ26&dNh zb+*06w_T!IJMIr;?^fI`0?ZCVFpNAr`9T>$EI-n%y|}SQ&$DG%ZBMf`Nm?LeKv6=H?bmMD zP@bJVz{Agf)fIpdEp9vGU0ZIlj=BNo19)^_8eRxs}#1Yte z!%uCtzvvqF3)+ZFknU)hP+_bAprA9KnAK_=yR}3YPdL4}7HGvf>`iQ4@Y|p66(BhL zHz~^DMGrk2mS-+zvb{{2A&guF{2oPZQ;x-DucK0P9>#IY^YE+M(bh;a%I`u^4h9lo z81XhMON|g@vU8E;HtH|C+sdSz1Lr3y<_iytsWtJM71$);O+U7bUg4bYEw*jnKz%hFzCVl#R)gO-~i$0Ymfy4{I`;35)(P9p3YL99ER z;}`1@-Y*^$gl~6yoygBU78E}c2BT8Y_h;Wk-=p(-fT}&Jy^)V3d2};sA536EB7h%@ zzK(SDL=W$jBSDk>{#x}uXI>p&NptrAXmsAyJH!(a;S|aU$*ZEus_Eud9=u=Em!4vO zy}@6*I~th8$(>4z=4R*B&+akFucDjhpR2yUC|o+-qJ|cniBeYNki(nZvYS{TKKPdU zWq7l1_x-+YKXTq zUTlT;kU)rPfov5;riJro!R9AS=$L$+uy}|A5cjUk1rw_zr=R(dAu5W5a4dW7Gv*GW zXUf?rc*Ai0Z%r{qpeGVwnY~8hp@jU=g3ON?`;Wz_O{fb`H#MQ`G$<+cuN1!1TSosR z@-sxvqBth>I6SV`e6qwSgJrIujG4yfni1e-q^jK5->)gMv09R!DUuEfILTjv18jP{ zT*)9xU?9hh2rU{mw6Qa_#vTD^S%X*@08NvK(Wxu%*fYjr4WT3(MR4g=i}C;)R?*P8H+9l3c28%Wg93*+@Ab&y{{0 zFO@ebfQq6QP+r_aj5ie1JbDAg$AgB%XXrZoCzwD*A}-XR06OU3I*yO7Bdj<&Q>k^r6<|m zX81*sv`DfD9fb^#pxMFA`MUf}cy3v>uEtw&*L6J}L^8jpPz>Yzs@n`2m(KRS6SqSj>cf zNJ1$lVV3Easa+7)TnM>ll;x6iJJeOj$fdDtj{sjv26Mcwa3CJV^NjI~St{S6){;;i zMcgK+=VgSnw8RJ>2IhKS`EJrO6_OdGU$M6jTkJA%!i4=NOJxgE_4pfoBWQ%@LRU%7 zS8(4RO?l6NEtUQvEFAH=SwClM5aUg3bAJ~_CD#m{&IBs zinslq?!$E0w!rUtt#LC?qYkQ#WPIpShitHTlD zJm^dy8*{s9GzQ1zpDbNk@_C_3|6dKAsL^o;h((e*JYsb1`t|GcDCzl7Y?4vXnGHzy zSlhCZ%a)zQmLsinsKrO75Lk(>o&IgkePMP;>8C@WOvKjHw8GTjLT9e(j)vC6BPTPJ z*z*X<>bwF^&99koa=`#a6OGXJYumSNTg$GZAd^*h`>bKb`Uzn)ji zw&-o-P90-!qs)R(p@V*|Q28a_N7Lglapi{>!^N#0Tl%pnPC-fiA~03xgY1I-iI zOPpRz9mgyHWPKio&ZPRA!>#qj#7yj~?F#^R$TMxH$;+P95angc67g`6dXLP5N)^mr zrGJ5GCWyV$mMpKY-9~i3zbn8EX}Qq}v0$C-V*v#dXCK|*YwV`OUvq3E{!U~#WR!Xf znb{0SC({=65hg^YMngu`gY@x$B0{&~o})F#L|tO&MS=0pdL9cwrvm*xHw7N{$ayuO z`-`bGMej(wqpheYOW688Bgdjx2GMGcu1&VnZY7$fUn)% z**%f)C)l>xb)9QE!R}mZl!t?(KV*3K#;Q#ZY3`Ls4Up1jXIkI0r5f{BRFGX%#Ud#m ziNtKbTUnvuiSiIf{dVo{*rz5Yek*!7dKKE6z%t#(OWhuNMIjp4W<*OO)91sz*w!-f zshS~6QDmq(Y#=}yPLYaSPs1znH7ldwrw3X+HTI#=epBb#na9QC5+H` zp&J%Sb&V!@8pv1UK0mi&24Nj0ZW>19zI^GyFM>eB*wt7OOFDNwW%uCF&L8xX2M_!> z`ESx1!}Wt5t$1J*eIVs$)WeE|&=OJmjY9BEYM328bisP#7S)Mtj7Z*qCY_E} zCMEyT_zcz@q~+ApcI@x8CeUj0YmDi$n z8q7u%lM+R|4F*cYm5R0TmJ^uxoZ~fVSkIpAtH@jh%J{&99!_|`8So~&lOpq0@tk+1 z>voQ}oLu+58@>!#3RHYp^=cn*?wMMtu6dMkk0cK?qV8@rCUY$ZTW|plAfwBN;ZgAfq7mJ4`FnZpsTp?ALV9N2`ZF|k&BG!;$cJ3#h|*$!3ENM zf;)&tkA%zB9AO+TW8apc%^D`;%!LaoGzML}c0Et;KPIanA|m1_?@fmPKsR0Nal7xD zS$?jK5PKDDc|ND_>#}O@|JV{_CR_+kAkP8XTSB?35>5Gah$KQ9u7F17e1u!uz^jc8 z_hbhaAsls)8CVDx@PL7BxP-#drbJw^veOA!(OdrknTGu|bdgaOiMhj||I{~HbF`FJI;0;i%#(2~qc5DN&S zEh=TP7_;lJU!oc`3)_xY6IEgM(U>$G^}0L`z=CU1PWLa++w`Y)C~$D=f^myZ8TpMK z=H-WSy(npZ(J~jBSpP<=Bb6MY|DS&#WF1z8 zf#gei`e8#mRL|2kL{ddswF(Tq#5_$7b)Fc**LuX4gW|LN3||Ky1Z@Kh^`On`>C}#U zQ4lB9fJN;Sc-LjVF^MC##smP*&bec6KxHE{w}v+}I~KLF)R}TIc!=Fe*WzC1kOj8! ze6mVL`%;J8ouiAGL6Gh6=Z<#30FF!nB}7F=2nXfv1@V~4 zl!Oeh-NaAGSLb*>Yr@!;9Yb5Ac=?U^c=!|k18yJclw@y!(DX@W<{D&68E{H>H|1Gy z-G5`?)~|h+f7>M{|6At0cuBUX{_R`0WQHKUSU2(uW$dd%DXJ6cK}c|?9T)o+>f=?+ zX;muvC%VcACjtGr1`)kPv%ruxVk0El5^TXXSNIIdd*Rb4zK-k22VDQx&S@5%lh>n< zTxv<45ZAIef-!zQrBbmhB-d52cgEgvs`E*zL4?^BGZU!+hYIJb8glmWk%JS~YK}!g zAbC4G-M3R zT?Ybc2U;L;tkgS%Ot!eWD4$G>=H$w9b0h(m##CHo^wCt)LLuQtNgS$t|9{YZMf|P% z!jfG>0b6~3^ubDrMxZVSPBjH`JBrtU=3RIXY6C)fJ5%8@yRex64HCs6`gCRkQ2gsE z(*Dy2eM2B%e}oml%q91XbU!kKLsZZ7%EbEDl5}0XA`*>AMU7uf3%QX*Xi}Y2JVCJ~ z-D#Eh8;f8gFw=Fy2{C2p9%2%nBnf(zs{kK4>(;LqAo1`_x|e?{FVp3LMRhKjDN1s{ zH3+~wl!F-zq8h;VA+N;3x z?|HOT>=2lsgB;N2gbe!jCbR2q{I#lJ%3<@Uymz-LK!OZ;Q(EZm#3ga@{{3-my+ie_ z!jon0H22^9I-4$5GSVd<{dfS5km{{I>rliS$!4c4+%ipGemN} zj*^4f0Xug_j5}tAd4nO=1e`U5Ptl?R^|^+6SzkQKotTcnB$Mwb^Y7ZHGTJ)L>$}bq zG?mLpYXjetzdIGw2&En!r^vx57(H0?@$;^;F4p<^rPn`EXaPJ@L&3hl_mDk4LD8i5 z*p`DgeyqH$7HhE@Hv;2)%2vPiS~@|&Q>RUPfB766vV<8?Bu5;d3vZ_C4vbTu4i^2T z(3NcZfrcp4z*&;!tPpOD`niQ`myR79P+U&q$nlym3+`tc(woFjpp0V(IJOaGpT}gb z>fO6RuFu}S?IKJhWm`@IKTVUbNP^k8fhWqLmHgS!T6XCIsY&|%QFQNCzIG1n%F0@Q zwMsneN7S@qbl){t7smF>Hv6qTCfm>BHz`!964OJDu9 z!yVjc-527`9Z{?pH?hi!ZjAx(s^EN$8~2=Rww9&`45}tfMgNs6&*jf1{o~W|oPsxx z#uv?$3kk$(DFPP zq(eZ{t-O9C21dssuoos>Xzt_a$t<20mZzO5zY~^Yyd^Vod2sQ?@ zhwgq17_utdacSrSSGV%UpPm)WbIv;Z`i+a9M&pN%KJ>{td%N`4;`=dgU2hdDN>?lM z{whj_DqR6h-07(#^t|}v#d_T3kAHJDPo#zSHH8oKUP-1QC-h64XEv^`F`PJo+^xrpC-=6ru?h0_)=ExI z?ZE2g)l$$)_vGM6>D<2k)#bl#-I|dZ^UGw;ER*^5V>TsdUOM%+ZYzI_&%QCZZF&U22Lx>P0i z^=seXCQ@jb{%pF9gXJ&+@t-KR5w=ttL+Nt*6sBgsj1k*x1SB4~*D%b&}u4 z>Z&SH(a||j_i`Nq^ZcJ#dHeZwnd35gY)CnFJ)O0G1s5c9?uip8#z>k(eMSz2vg(4TbPXX1`Zed-#j(UeIiaob7@xv4 zM!8CL>(;&XT_GtY`ySVLGgHT{WMMOQdz$5+XO?2(2|0CY+_oKi_Jop~or9(~6)l(Q zDtGc!nnZhY?iLgjWW)b#3xoz+_vpDJ3Y=?KuO=+I-7o9=R9J_6gsg^+1hM7=Um3=J z`t+#-AAVYjH4lP}Ljr->r!$`Ch4T`4r`wS7$AcE=Q${kaq`LvLJQ~-@VRla%K)rRY zb6*{sF=HG!Q*&Ov95sJ_q;}J$m(qx&=H?I6UHSQ7VBWAq*<;|K1~tose#rgajtf{+ zTVKEhl?@<=TEZ_C(YhJ&7 z`vmDfcW=biwlL1WGjQkb-LqN5y|9#wC&G|R-6xkQkXszpH!}0ajvl=e3&=h2ove<= zE}wrR9eHnRZWM?q3*^DtL4PpI%i&D!ku!+)mbbae(|l8YH)E0Y4L@?^ zT(-lgF=J+F^nG_eutMi>{p!_jk*N+wezz%;Bch{I#+Bjyv|YIH=#1&p$C6>#ukhLS zse}Hwc>}h+Q;QZCDAMN>LwXcJ%G!pDj9rV4KjJ-2jhs4e+@51Dh!T1cBQ$K}$P@U+ zJ`|TuU6+oK%xo^2v=<)=f7;fs_|>uAeLq5@kA*krH1;=y#fgX6OIqEBg1Yf4z2&pi zk8Ze)`jzJ2z4aEu=y5)xydF3)A7D$E`l&tl*@D4bZCOY&X3n&=vkMsV`%)EPcUj1gWt!}X|h8u~d?}Y*E5t}|AbHhhSm>;fV2%inBWxxek8JiAj z=*Uuv=CJ(jHGa(pB`aeWZD#$BnEG~<)_baqKCj89m@_Fd713=on_pR($4FKi$HsQ~ zR1e?G*vHcc{eeHue%s-L2cPVW{;^3}nQ9&pv1d*lb*q2l(AWbl)|*&mWwvXjEL@$Z zGOy9dtfkF0@6f`sH@A*WH=epLw!CaSzD%HEkC5;lJ$nZ7vABzalVbL48nrN&q-^^aeh9^vD*IGCh5lpy1#i9dQIU_lnP)5Z&9G316(J z_As+^C(bG99!ifhd+ywy*h=p@dmiC2388u(j`TpQc=PD4jjd39w1^?N@Y z@$&sOoXihwP0ME>$6W|IT~&Uh z`4-$OBQRowAlO^RRanujMh2U&6<^``u)h0q<)a)9G=b^6&L-EL+{wyXwd5 zZC$o{U+b~o+w^tHbgZfJP-nBof*!Z#D++_R0|1Od9(*XgAQzU;c;lNd|Jc>WW#xk2 zy+6*MFmd&m$_wpb_3!_9v+|cH?!`CuWLNxhMah^~b+r^ttN&krDzA1~WpUQBy7zDL zrVJJj8oPMld`HI_^A;-u=g(U*> or <> configuration sections, and to the last response headers when used in <>, <> or <> configuration sections. +- `body`: A map containing the body. References request body when used in <> configuration section, and to the last response body when used in <>, <> or <> configuration sections. +- `cursor`: A map containing any data the user configured to be stored between restarts (See <>). -All of the mentioned objects are only stored at runtime, except `cursor` which values are persisted between restarts. +All of the mentioned objects are only stored at runtime, except `cursor`, which has values that are persisted between restarts. +[[transforms]] ==== Transforms -A transform is an action that let the user modify the input state. Depending on where the transform is defined, it will have access for reading or writing different elements of the state. -This access limitations will be specified in the corresponding configuration sections. +A transform is an action that lets the user modify the <>. Depending on where the transform is defined, it will have access for reading or writing different elements of the <>. + +The access limitations are described in the corresponding configuration sections. [float] ==== `append` -Append will append a value to a list. If the field does not exist the first entry will be a scalar value, and subsequent additions will convert it to a list. +Appends a value to a list. If the field does not exist, the first entry will be a scalar value, and subsequent additions will convert the value to a list. ["source","yaml",subs="attributes"] ---- @@ -119,13 +132,13 @@ Append will append a value to a list. If the field does not exist the first entr ---- - `target` defines the destination field where the value is stored. -- `value` defines the value that will be stored and it is a value template. +- `value` defines the value that will be stored and it is a <>. - `default` defines the fallback value whenever `value` is empty or the template parsing fails. Default templates do not have access to any state, only to functions. [float] ==== `delete` -Delete will delete the target field. +Deletes the target field. ["source","yaml",subs="attributes"] ---- @@ -133,14 +146,12 @@ Delete will delete the target field. target: body.foo.bar ---- -- `target` defines the destination field to delete. - -NOTE: If `target` is a list, it will delete the list completely, and not a single element. +- `target` defines the destination field to delete. If `target` is a list and not a single element, the complete list will be deleted. [float] ==== `set` -Set will set a value. +Sets a value. ["source","yaml",subs="attributes"] ---- @@ -151,14 +162,15 @@ Set will set a value. ---- - `target` defines the destination field where the value is stored. -- `value` defines the value that will be stored and it is a value template. +- `value` defines the value that will be stored and it is a <>. - `default` defines the fallback value whenever `value` is empty or the template parsing fails. Default templates do not have access to any state, only to functions. +[[value-templates]] ==== Value templates -Some configuration options and transforms can use value templates. Value templates are Go templates with access to the input state and to some built in functions. +Some configuration options and transforms can use value templates. Value templates are Go templates with access to the input state and to some built-in functions. -The state elements and what operations can be performed are defined by the option or transform using them and will be specified in them. +To see which <> and operations are available, see the documentation for the option or <> where you want to use a value template. A value template looks like: @@ -170,19 +182,19 @@ A value template looks like: default: "a default value" ---- -What is between `{{` `}}` will be evaluated. For more information on Go templates please refer to https://golang.org/pkg/text/template[the Go docs]. +The content inside the curly braces `{{` `}}` is evaluated. For more information on Go templates please refer to https://golang.org/pkg/text/template[the Go docs]. -Some built in helper functions are provided to work with the input state inside value templates: +Some built-in helper functions are provided to work with the input state inside value templates: - `parseDuration`: parses duration strings and returns `time.Duration`. Example: `{{parseDuration "1h"}}`. - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `{{now (parseDuration "-1h")}}` returns the time at 1 hour before now. - `parseTimestamp`: parses a timestamp in seconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732}}` returns `2020-11-05 13:25:32 +0000 UTC`. - `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732000}}` returns `2020-11-05 13:25:32 +0000 UTC`. - `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `{{parseTimestamp 1604582732000000000}}` returns `2020-11-05 13:25:32 +0000 UTC`. -- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Go lang predefined layouts or a custom one. Example: `{{ parseDate "2020-11-05T12:25:32Z" }}`, `{{ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" }}`, `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC }}`. -- `formatDate`: formats a `time.Time`. By default the format layout is `RFC3339` but optionally can accept any of the Go lang predefined layouts or a custom one. It will default to UTC timezone when formatting but optionally a different timezone can be specified. If the timezone is incorrect will default to UTC. Example: `{{ formatDate (now) "UnixDate" }}`, `{{ formatDate (now) "UnixDate" "America/New_York" }}`. -- `getRFC5988Link`: it extracts a specific relation from a list of https://tools.ietf.org/html/rfc5988[RFC5988] links. It is useful when we are parsing header values for pagination, for example. Example: `{{ getRFC5988Link "next" .last_response.header.Link }}` -- `toInt`: converts a string to an integer, returns 0 if it fails. +- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `{{ parseDate "2020-11-05T12:25:32Z" }}`, `{{ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" }}`, `{{ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC }}`. +- `formatDate`: formats a `time.Time`. By default the format layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. It will default to UTC timezone when formatting, but you can specify a different timezone. If the timezone is incorrect, it will default to UTC. Example: `{{ formatDate (now) "UnixDate" }}`, `{{ formatDate (now) "UnixDate" "America/New_York" }}`. +- `getRFC5988Link`: extracts a specific relation from a list of https://tools.ietf.org/html/rfc5988[RFC5988] links. It is useful when parsing header values for pagination. Example: `{{ getRFC5988Link "next" .last_response.header.Link }}`. +- `toInt`: converts a string to an integer. Returns 0 if the conversion fails. - `add`: adds a list of integers and returns their sum. In addition to the provided functions, any of the native functions for `time.Time` and `http.Header` types can be used on the corresponding objects. Examples: `{{(now).Day}}`, `{{.last_response.header.Get "key"}}` @@ -197,7 +209,7 @@ The `httpjson` input supports the following configuration options plus the Defines the configuration version. Current supported versions are: `1` and `2`. Default: `1`. -NOTE: This defaulting to `1` is just to avoid breaking current configurations. V1 configuration is deprecated and will be unsupported in next releases. Any new configuration should use `config_version: 2`. +NOTE: This setting defaults to `1` to avoid breaking current configurations. V1 configuration is deprecated and will be unsupported in future releases. Any new configuration should use `config_version: 2`. [float] ==== `interval` @@ -207,8 +219,7 @@ Duration between repeated requests. It may make additional pagination requests i [float] ==== `auth.basic.enabled` -The `enabled` setting can be used to disable the basic auth configuration by -setting it to `false`. Default: `true`. +When set to `false`, disables the basic auth configuration. Default: `true`. NOTE: Basic auth settings are disabled if either `enabled` is set to `false` or the `auth.basic` section is missing. @@ -216,18 +227,17 @@ the `auth.basic` section is missing. [float] ==== `auth.basic.user` -The `user` setting sets the user to authenticate with. +The user to authenticate with. [float] ==== `auth.basic.password` -The `password` setting sets the password to use. +The password to use. [float] ==== `auth.oauth2.enabled` -The `enabled` setting can be used to disable the oauth2 configuration by -setting it to `false`. The default value is `true`. +When set to `false`, disables the oauth2 configuration. Default: `true`. NOTE: OAuth2 settings are disabled if either `enabled` is set to `false` or the `auth.oauth2` section is missing. @@ -235,41 +245,39 @@ the `auth.oauth2` section is missing. [float] ==== `auth.oauth2.provider` -The `provider` setting can be used to configure supported oauth2 providers. +Used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. Supported providers are: `azure`, `google`. [float] ==== `auth.oauth2.client.id` -The `client.id` setting is used as part of the authentication flow. It is always required +The client ID used as part of the authentication flow. It is always required except if using `google` as provider. Required for providers: `default`, `azure`. [float] ==== `auth.oauth2.client.secret` -The `client.secret` setting is used as part of the authentication flow. It is always required +The client secret used as part of the authentication flow. It is always required except if using `google` as provider. Required for providers: `default`, `azure`. [float] ==== `auth.oauth2.scopes` -The `scopes` setting defines a list of scopes that will be requested during the oauth2 flow. +A list of scopes that will be requested during the oauth2 flow. It is optional for all providers. [float] ==== `auth.oauth2.token_url` -The `token_url` setting specifies the endpoint that will be used to generate the -tokens during the oauth2 flow. It is required if no provider is specified. +The endpoint that will be used to generate the tokens during the oauth2 flow. It is required if no provider is specified. NOTE: For `azure` provider either `token_url` or `azure.tenant_id` is required. [float] ==== `auth.oauth2.endpoint_params` -The `endpoint_params` setting specifies a set of values that will be sent on each -request to the `token_url`. Each param key can have multiple values. +Set of values that will be sent on each request to the `token_url`. Each param key can have multiple values. Can be set for all providers except `google`. ["source","yaml",subs="attributes"] @@ -288,7 +296,7 @@ Can be set for all providers except `google`. [float] ==== `auth.oauth2.azure.tenant_id` -The `azure.tenant_id` is used for authentication when using `azure` provider. +Used for authentication when using `azure` provider. Since it is used in the process to generate the `token_url`, it can't be used in combination with it. It is not required. @@ -298,13 +306,13 @@ https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-ser [float] ==== `auth.oauth2.azure.resource` -The `azure.resource` is used to identify the accessed WebAPI resource when using `azure` provider. +The accessed WebAPI resource when using `azure` provider. It is not required. [float] ==== `auth.oauth2.google.credentials_file` -The `google.credentials_file` setting specifies the credentials file for Google. +The credentials file for Google. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading default credentials from the environment will be attempted via ADC. For more information about @@ -313,7 +321,7 @@ how to provide Google credentials, please refer to https://cloud.google.com/docs [float] ==== `auth.oauth2.google.credentials_json` -The `google.credentials_json` setting allows to write your credentials information as raw JSON. +Your credentials information as raw JSON. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading default credentials from the environment will be attempted via ADC. For more information about @@ -322,7 +330,7 @@ how to provide Google credentials, please refer to https://cloud.google.com/docs [float] ==== `auth.oauth2.google.jwt_file` -The `google.jwt_file` setting specifies the JWT Account Key file for Google. +The JWT Account Key file for Google. NOTE: Only one of the credentials settings can be set at once. If none is provided, loading default credentials from the environment will be attempted via ADC. For more information about @@ -373,48 +381,50 @@ information. [float] ==== `request.retry.max_attempts` -This specifies the maximum number of retries for the HTTP client. Default: `5`. +The maximum number of retries for the HTTP client. Default: `5`. [float] ==== `request.retry.wait_min` -This specifies the minimum time to wait before a retry is attempted. Default: `1s`. +The minimum time to wait before a retry is attempted. Default: `1s`. [float] ==== `request.retry.wait_max` -This specifies the maximum time to wait before a retry is attempted. Default: `60s`. +The maximum time to wait before a retry is attempted. Default: `60s`. [float] ==== `request.redirect.forward_headers` -This specifies if headers are forwarded in case of a redirect. Default: `false`. +When set to `true` request headers are forwarded in case of a redirect. Default: `false`. [float] ==== `request.redirect.headers_ban_list` -When `redirect.forward_headers` is set to `true`, all headers __except__ the ones defined in this list, will be forwarded. Default: `[]`. +When `redirect.forward_headers` is set to `true`, all headers __except__ the ones defined in this list will be forwarded. Default: `[]`. [float] ==== `request.redirect.max_redirects` -Sets the maximum number of redirects to follow for a request. Default: `10`. +The maximum number of redirects to follow for a request. Default: `10`. +[[request-rate-limit]] [float] ==== `request.rate_limit.limit` -This specifies the value of the response that specifies the total limit. It is defined with a Go template value. Can read state from: [`.last_response.header`] +The value of the response that specifies the total limit. It is defined with a Go template value. Can read state from: [`.last_response.header`] [float] ==== `request.rate_limit.remaining` -This specifies the value of the response that specifies the remaining quota of the rate limit. It is defined with a Go template value. Can read state from: [`.last_response.header`] +The value of the response that specifies the remaining quota of the rate limit. It is defined with a Go template value. Can read state from: [`.last_response.header`] [float] ==== `request.rate_limit.reset` -This specifies the value of the response that specifies the epoch time when the rate limit will reset. It is defined with a Go template value. Can read state from: [`.last_response.header`] +The value of the response that specifies the epoch time when the rate limit will reset. It is defined with a Go template value. Can read state from: [`.last_response.header`] +[[request-transforms]] [float] ==== `request.transforms` @@ -439,6 +449,7 @@ filebeat.inputs: value: '{{now (parseDuration "-1h")}}' ---- +[[response-transforms]] [float] ==== `response.transforms` @@ -474,6 +485,7 @@ filebeat.inputs: value: 5m ---- +[[response-split]] [float] ==== `response.split` @@ -510,17 +522,18 @@ If set to true, the fields from the parent document (at the same level as `targe [float] ==== `response.split[].key_field` -It can only be used with `type: map`. When not empty, will define the a new field where the original key value will be stored. +Valid when used with `type: map`. When not empty, defines a new field where the original key value will be stored. [float] ==== `response.split[].split` -Nested split operation. Split operations can be nested at will, an event won't be created until the deepest split operation is applied. +Nested split operation. Split operations can be nested at will. An event won't be created until the deepest split operation is applied. +[[response-pagination]] [float] ==== `response.pagination` -List of transforms to apply to the response to every new page request. All the transforms from `request.transform` will be executed and then `response.pagination` will be added to modify the next request as needed. For subsequent responses, the usual `response.transforms` and `response.split` will be executed normally. +List of transforms that will be applied to the response to every new page request. All the transforms from `request.transform` will be executed and then `response.pagination` will be added to modify the next request as needed. For subsequent responses, the usual `response.transforms` and `response.split` will be executed normally. Available transforms for pagination: [`append`, `delete`, `set`]. @@ -530,8 +543,9 @@ Can write state to: [`body.*`, `header.*`, `url.*`]. Examples using split: -- We have a response with two nested arrays and we want a document for each of the elements of the inner array: +- We have a response with two nested arrays, and we want a document for each of the elements of the inner array: ++ ["source","json",subs="attributes"] ---- { @@ -563,8 +577,10 @@ Examples using split: } ---- -Our config will look like ++ +The config will look like: ++ ["source","yaml",subs="attributes"] ---- filebeat.inputs: @@ -583,8 +599,10 @@ filebeat.inputs: keep_parent: true ---- ++ This will output: ++ ["source","json",subs="attributes"] ---- [ @@ -627,8 +645,9 @@ This will output: ] ---- -- We have a response with two an array with objects, and we want a document for each of the object keys while keeping the keys values: +- We have a response with an array with two objects, and we want a document for each of the object keys while keeping the keys values: ++ ["source","json",subs="attributes"] ---- { @@ -654,8 +673,10 @@ This will output: } ---- -Our config will look like ++ +The config will look like: ++ ["source","yaml",subs="attributes"] ---- filebeat.inputs: @@ -675,8 +696,10 @@ filebeat.inputs: key_field: id ---- ++ This will output: ++ ["source","json",subs="attributes"] ---- [ @@ -697,8 +720,9 @@ This will output: ] ---- -- We have a response with two an array with objects, and we want a document for each of the object keys while applying a transform to each: +- We have a response with an array with two objects, and we want a document for each of the object keys while applying a transform to each: ++ ["source","json",subs="attributes"] ---- { @@ -724,8 +748,10 @@ This will output: } ---- -Our config will look like ++ +The config will look like: ++ ["source","yaml",subs="attributes"] ---- filebeat.inputs: @@ -745,8 +771,10 @@ filebeat.inputs: type: map ---- ++ This will output: ++ ["source","json",subs="attributes"] ---- [ @@ -761,10 +789,11 @@ This will output: ] ---- +[[cursor]] [float] ==== `cursor` -Cursor is a list of key value objects where an arbitrary values are defined. The values are interpreted as value templates and a default template can be set. Cursor state is kept between input restarts and updated once all the events for a request are published. +Cursor is a list of key value objects where arbitrary values are defined. The values are interpreted as <> and a default template can be set. Cursor state is kept between input restarts and updated once all the events for a request are published. Can read state from: [`.last_response.*`, `.last_event.*`]. @@ -840,6 +869,7 @@ Deprecated, use `response.pagination`. ==== `rate_limit.*` Deprecated, use `request.rate_limit.*`. + [float] ==== `retry.*` @@ -862,47 +892,15 @@ Deprecated, use `auth.oauth2.*`. ==== Request life cycle +image:images/input-httpjson-lifecycle.png[Request lifecycle] -.... -+-------+ +-------------------+ +---------------------+ +---------------------------+ +---------------+ +---------+ -| Input | | RequestTransforms | | ResponsePagination | | ResponseTransforms/Split | | RemoteServer | | Output | -+-------+ +-------------------+ +---------------------+ +---------------------------+ +---------------+ +---------+ - | | | | | | - | 1. At interval, | | | | | - | create a new request. | | | | | - |------------------------------->| | | | | - | ---------------------------\ | | | | | - | | 2. Transform the request |-| | | | | - | | before executing it. | | | | | | - | |--------------------------| | | | | | - | | 3. Execute request. | | | | - | |--------------------------------------------------------------------------------------->| | - | | | | | | - | | | | 4. Return response. | | - | | | |<---------------------------| | - | | | -------------------------\ | | | - | | | | 5. Transform response |-| | | - | | | | into a list of events. | | | | - | | | |------------------------| | | | - | | | | 6. Publish every | | - | | | | event to output. | | - | | | |------------------------------------------>| - | | | -----------------------------\ | | | - | | |-| 7. If there are more pages | | | | - | | | | transform the request. | | | | - | | | |----------------------------| | | | - | | | | | | - | | | Execute request and go back to 4. | | | - | | |---------------------------------------------------------------->| | -.... - -. At every defined interval a new request will be created. -. The request will be transformed using the configured `request.transforms`. -. The resulting transformed request will be executed. -. The server will respond (here is where any retry or rate limit policy will take place when configured). -. The response will be transformed using the configured `response.transforms` and `response.split`. -. Each resulting event will be published to the output. -. If a `response.pagination` is configured and there are more pages, a new request will be created using it, otherwise the process ends until the next interval. +. At every defined interval a new request is created. +. The request is transformed using the configured `request.transforms`. +. The resulting transformed request is executed. +. The server responds (here is where any retry or rate limit policy takes place when configured). +. The response is transformed using the configured `response.transforms` and `response.split`. +. Each resulting event is published to the output. +. If a `response.pagination` is configured and there are more pages, a new request is created using it, otherwise the process ends until the next interval. [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go index 729ff417f46..bec2991a71b 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go @@ -5,6 +5,8 @@ package v2 import ( + "go.uber.org/multierr" + "github.com/elastic/go-concert/unison" v2 "github.com/elastic/beats/v7/filebeat/input/v2" @@ -42,7 +44,10 @@ func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { registerRequestTransforms() registerResponseTransforms() registerPaginationTransforms() - return m.stateless.Init(grp, mode) // multierr.Append() + return multierr.Append( + m.stateless.Init(grp, mode), + m.cursor.Init(grp, mode), + ) } // Create creates a cursor input manager if the config has a date cursor set up,