From 95ba8f246da6d8ef54447a3f894977fc1d4c8ced Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 10 Sep 2021 15:35:56 +0000 Subject: [PATCH 1/7] Init commit --- go.mod | 1 + go.sum | 12 ++++ .../input/httpjson/internal/v2/config_auth.go | 56 ++++++++++++++++++- .../input/httpjson/internal/v2/pagination.go | 5 +- .../input/httpjson/internal/v2/request.go | 36 ++++++++---- 5 files changed, 95 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 07baf547449f..aa650033c71e 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/Shopify/sarama v1.27.0 github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc + github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/akavel/rsrc v0.8.0 // indirect github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d diff --git a/go.sum b/go.sum index c14acc22e90a..5eaca9c93a5c 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 h1:9OmEpkkO4v github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc h1:9iW/Fbn/R/nyUOiqo6AgwBe8uirqUIoTGF3vKG8qjoc= github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= +github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs= +github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -389,6 +391,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -426,6 +429,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0 h1:sBDQoHXrOlfPobnKw69FIKa1wg9qsL github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/cronexpr v1.1.0 h1:dnNsWtH0V2ReN7JccYe8m//Bj14+PjJDntR1dz0Cixk= github.com/hashicorp/cronexpr v1.1.0/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -585,6 +589,7 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -624,6 +629,8 @@ github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= @@ -694,6 +701,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -756,6 +764,7 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 h1:0gYLpmzecnaDCoeWxSfEJ7J1b6B/67+NV++4HKQXx+Y= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= @@ -998,8 +1007,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go b/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go index 0d862ea181b2..b0ae89c0cd89 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go @@ -19,16 +19,27 @@ import ( "golang.org/x/oauth2/endpoints" "golang.org/x/oauth2/google" + "github.com/akamai/AkamaiOPEN-edgegrid-golang/edgegrid" "github.com/elastic/beats/v7/libbeat/common" ) type authConfig struct { - Basic *basicAuthConfig `config:"basic"` - OAuth2 *oAuth2Config `config:"oauth2"` + Basic *basicAuthConfig `config:"basic"` + OAuth2 *oAuth2Config `config:"oauth2"` + EdgeGrid *edgeGridConfig `config:"edgegrid"` } func (c authConfig) Validate() error { - if c.Basic.isEnabled() && c.OAuth2.isEnabled() { + list := []bool{c.Basic.isEnabled(), c.OAuth2.isEnabled(), c.EdgeGrid.isEnabled()} + filtered := []bool{} + + for i := range list { + if list[i] { + filtered = append(filtered, list[i]) + } + } + + if len(filtered) > 1 { return errors.New("only one kind of auth can be enabled") } return nil @@ -263,3 +274,42 @@ func (o *oAuth2Config) validateAzureProvider() error { return nil } + +type edgeGridConfig struct { + Enabled *bool `config:"enabled"` + Host string `config:"host"` + ClientToken string `config:"client_token"` + ClientSecret string `config:"client_secret"` + AccessToken string `config:"access_token"` +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (e *edgeGridConfig) isEnabled() bool { + return e != nil && (e.Enabled == nil || *e.Enabled) +} + +// Validate checks if oauth2 config is valid. +func (e *edgeGridConfig) Validate() error { + if !e.isEnabled() { + return nil + } + + if e.Host == "" || e.ClientToken == "" || e.ClientSecret == "" || e.AccessToken == "" { + return errors.New("All parameters must be set; Host, Client Token, Client Secret, Access Token.") + } + + return nil +} + +func (e *edgeGridConfig) config() edgegrid.Config { + config := edgegrid.Config{ + Host: e.Host, + ClientToken: e.ClientToken, + ClientSecret: e.ClientSecret, + AccessToken: e.AccessToken, + MaxBody: 1024, + HeaderToSign: []string{}, + Debug: false, + } + return config +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index 6ea063a10af0..8615e5a04503 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -78,6 +78,9 @@ func newPaginationRequestFactory(method, encodeAs string, url url.URL, body *com rf.user = authConfig.Basic.User rf.password = authConfig.Basic.Password } + if authConfig != nil && authConfig.EdgeGrid.isEnabled() { + rf.EdgeGridConfig = authConfig.EdgeGrid.config() + } return rf } @@ -135,7 +138,7 @@ func (iter *pageIterator) next() (*response, bool, error) { return nil, false, err } - resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) + resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq, iter.pagination.requestFactory) if err != nil { return nil, false, err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index 921b13b9ab71..4baafbc06751 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -12,6 +12,8 @@ import ( "net/http" "net/url" + egclient "github.com/akamai/AkamaiOPEN-edgegrid-golang/client-v1" + "github.com/akamai/AkamaiOPEN-edgegrid-golang/edgegrid" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -30,9 +32,14 @@ type httpClient struct { limiter *rateLimiter } -func (c *httpClient) do(stdCtx context.Context, trCtx *transformContext, req *http.Request) (*http.Response, error) { +func (c *httpClient) do(stdCtx context.Context, trCtx *transformContext, req *http.Request, rf *requestFactory) (*http.Response, error) { resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { - return c.client.Do(req) + + if rf.EdgeGridConfig.Host != "" { + return egclient.Do(rf.EdgeGridConfig, req) + } else { + return c.client.Do(req) + } }) if err != nil { return nil, fmt.Errorf("failed to execute http client.Do: %w", err) @@ -80,14 +87,15 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro } type requestFactory struct { - url url.URL - method string - body *common.MapStr - transforms []basicTransform - user string - password string - log *logp.Logger - encoder encoderFunc + url url.URL + method string + body *common.MapStr + transforms []basicTransform + user string + password string + log *logp.Logger + encoder encoderFunc + EdgeGridConfig edgegrid.Config } func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { @@ -105,6 +113,9 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. rf.user = authConfig.Basic.User rf.password = authConfig.Basic.Password } + if authConfig != nil && authConfig.EdgeGrid.isEnabled() { + rf.EdgeGridConfig = authConfig.EdgeGrid.config() + } return rf } @@ -128,6 +139,9 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx *transfor url := trReq.url() req, err := http.NewRequest(rf.method, url.String(), bytes.NewBuffer(body)) + if rf.EdgeGridConfig.Host != "" { + req, err = egclient.NewRequest(rf.EdgeGridConfig, rf.method, url.String(), nil) + } if err != nil { return nil, err } @@ -169,7 +183,7 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p return fmt.Errorf("failed to create http request: %w", err) } - httpResp, err := r.client.do(stdCtx, trCtx, req) + httpResp, err := r.client.do(stdCtx, trCtx, req, r.requestFactory) if err != nil { return fmt.Errorf("failed to execute http client.Do: %w", err) } From ac14e79af120802ecf1a051c9be33c0fee004f9c Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 10 Sep 2021 18:40:56 +0000 Subject: [PATCH 2/7] Update docs and tests --- CHANGELOG.next.asciidoc | 2 + .../docs/inputs/input-httpjson.asciidoc | 44 ++++++++++++- .../input/httpjson/internal/v2/config_test.go | 66 +++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0d60df49c5ae..86b1ede2652b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -756,6 +756,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `ignore_empty_value` flag to `httpjson` `split` processor. {pull}27880[27880] - Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879] - Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937] +- Add Akamai EdgeGrid authenication to `httpjson` input {pull}27873[27873] + *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 0585f10d46e3..2d3e22c9d58f 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -61,7 +61,7 @@ filebeat.inputs: value: 5m ---- -Additionally, it supports authentication via Basic auth, HTTP Headers or oauth2. +Additionally, it supports authentication via Basic auth, HTTP Headers, oauth2, or Akamai EdgeGrid. Example configurations with authentication: @@ -89,6 +89,19 @@ filebeat.inputs: request.url: http://localhost ---- +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: httpjson + config_version: 2 + auth.edgegrid: + client_token: 12345678901234567890abcdef + client_secret: abcdef12345678901234567890 + access_token: 1qaz2wsx3edc4rfv + host: localhost + request.url: http://localhost +---- + [[input-state]] ==== Input state @@ -347,6 +360,35 @@ NOTE: Only one of the credentials settings can be set at once. If none is provid default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. +[float] +==== `auth.edgegrid.enabled` + +When set to `false`, disables the EdgeGrid auth configuration. Default: `true`. + +NOTE: EdgeGrid auth settings are disabled if either `enabled` is set to `false` or +the `auth.edgegrid` section is missing. + +[float] +==== `auth.edgegrid.client_token` + +Client Token generated from https://control.akamai.com/ + +[float] +==== `auth.edgegrid.client_secret` + +Client Secret generated from https://control.akamai.com/ + +[float] +==== `auth.edgegrid.access_token` + +Access Token generated from https://control.akamai.com/ + +[float] +==== `auth.edgegrid.host` + +Unique API host generated from https://control.akamai.com/. In the format of `akaa-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net` +with no path or scheme + [float] ==== `request.url` diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go index 4110430b25e8..681bfb803795 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go @@ -392,3 +392,69 @@ func TestCursorEntryConfig(t *testing.T) { assert.True(t, conf["entry3"].mustIgnoreEmptyValue()) assert.True(t, conf["entry4"].mustIgnoreEmptyValue()) } + +func TestConfigEdgeGridValidation(t *testing.T) { + cases := []struct { + name string + expectedErr string + input map[string]interface{} + setup func() + teardown func() + }{ + { + name: "can't set edge grid and basic auth together", + expectedErr: "only one kind of auth can be enabled accessing 'auth'", + input: map[string]interface{}{ + "auth.basic.user": "user", + "auth.basic.password": "pass", + "auth.edgegrid": map[string]interface{}{ + "client_token": "asdf", + "host": "localhost", + "client_secret": "asdf", + "access_token": "asdf", + }, + }, + }, + { + name: "Must set all config parameters", + expectedErr: "All parameters must be set; Host, Client Token, Client Secret, Access Token. accessing 'auth.edgegrid'", + input: map[string]interface{}{ + "auth.edgegrid": map[string]interface{}{ + "client_token": "asdf", + "host": "localhost", + "client_secret": "asdf", + }, + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + if c.setup != nil { + c.setup() + } + + if c.teardown != nil { + defer c.teardown() + } + + c.input["request.url"] = "localhost" + cfg := common.MustNewConfigFrom(c.input) + conf := defaultConfig() + err := cfg.Unpack(&conf) + + switch { + case c.expectedErr == "": + if err != nil { + t.Fatalf("Configuration validation failed. no error expected but got %q", err) + } + + case c.expectedErr != "": + if err == nil || err.Error() != c.expectedErr { + t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) + } + } + }) + } +} From 7efbbf4444aecc7907f1a0b5ce22878fb006701c Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 14 Sep 2021 18:34:58 +0000 Subject: [PATCH 3/7] Add Base64 HMAC --- .../input/httpjson/internal/v2/value_tpl.go | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index 9f91fcdccce5..ed00cf97bbef 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -55,7 +55,8 @@ func (t *valueTpl) Unpack(in string) error { "add": add, "mul": mul, "div": div, - "hmac": hmacString, + "hmac": hmacStringHex, + "hmacBase64": hmacStringBase64, "base64Encode": base64Encode, "base64EncodeNoPad": base64EncodeNoPad, "join": strings.Join, @@ -267,10 +268,9 @@ func base64EncodeNoPad(values ...string) string { return base64.RawStdEncoding.EncodeToString([]byte(data)) } -func hmacString(hmacType string, hmacKey string, values ...string) string { - data := strings.Join(values[:], "") +func hmacString(hmacType string, hmacKey string, data string) []byte { if data == "" { - return "" + return nil } // Create a new HMAC by defining the hash type and the key (as byte array) var mac hash.Hash @@ -281,11 +281,32 @@ func hmacString(hmacType string, hmacKey string, values ...string) string { mac = hmac.New(sha1.New, []byte(hmacKey)) default: // Upstream config validation prevents this from happening. - return "" + return nil } // Write Data to it mac.Write([]byte(data)) + // Get result and encode as bytes + return mac.Sum(nil) +} + +func hmacStringHex(hmacType string, hmacKey string, values ...string) string { + data := strings.Join(values[:], "") + if data == "" { + return "" + } + bytes := hmacString(hmacType, hmacKey, data) + // Get result and encode as hexadecimal string + return hex.EncodeToString(bytes) +} + +func hmacStringBase64(hmacType string, hmacKey string, values ...string) string { + data := strings.Join(values[:], "") + if data == "" { + return "" + } + bytes := hmacString(hmacType, hmacKey, data) + // Get result and encode as hexadecimal string - return hex.EncodeToString(mac.Sum(nil)) + return base64.StdEncoding.EncodeToString(bytes) } From 061f019c087a073346d943402ab533bd94dc95a6 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 15 Sep 2021 12:48:31 +0000 Subject: [PATCH 4/7] Remove EdgeGrid Library --- go.mod | 1 - go.sum | 11 ---- .../docs/inputs/input-httpjson.asciidoc | 44 +------------ .../input/httpjson/internal/v2/config_auth.go | 56 +--------------- .../input/httpjson/internal/v2/config_test.go | 66 ------------------- .../input/httpjson/internal/v2/pagination.go | 5 +- .../input/httpjson/internal/v2/request.go | 36 ++++------ 7 files changed, 16 insertions(+), 203 deletions(-) diff --git a/go.mod b/go.mod index aa650033c71e..07baf547449f 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/Shopify/sarama v1.27.0 github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc - github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect github.com/akavel/rsrc v0.8.0 // indirect github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d diff --git a/go.sum b/go.sum index 5eaca9c93a5c..793736682864 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,6 @@ github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 h1:9OmEpkkO4v github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc h1:9iW/Fbn/R/nyUOiqo6AgwBe8uirqUIoTGF3vKG8qjoc= github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= -github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs= -github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -391,7 +389,6 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -429,7 +426,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0 h1:sBDQoHXrOlfPobnKw69FIKa1wg9qsL github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/cronexpr v1.1.0 h1:dnNsWtH0V2ReN7JccYe8m//Bj14+PjJDntR1dz0Cixk= github.com/hashicorp/cronexpr v1.1.0/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -589,7 +585,6 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -629,8 +624,6 @@ github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= @@ -701,7 +694,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -1007,11 +999,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= -gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 2d3e22c9d58f..0585f10d46e3 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -61,7 +61,7 @@ filebeat.inputs: value: 5m ---- -Additionally, it supports authentication via Basic auth, HTTP Headers, oauth2, or Akamai EdgeGrid. +Additionally, it supports authentication via Basic auth, HTTP Headers or oauth2. Example configurations with authentication: @@ -89,19 +89,6 @@ filebeat.inputs: request.url: http://localhost ---- -["source","yaml",subs="attributes"] ----- -filebeat.inputs: -- type: httpjson - config_version: 2 - auth.edgegrid: - client_token: 12345678901234567890abcdef - client_secret: abcdef12345678901234567890 - access_token: 1qaz2wsx3edc4rfv - host: localhost - request.url: http://localhost ----- - [[input-state]] ==== Input state @@ -360,35 +347,6 @@ NOTE: Only one of the credentials settings can be set at once. If none is provid default credentials from the environment will be attempted via ADC. For more information about how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication. -[float] -==== `auth.edgegrid.enabled` - -When set to `false`, disables the EdgeGrid auth configuration. Default: `true`. - -NOTE: EdgeGrid auth settings are disabled if either `enabled` is set to `false` or -the `auth.edgegrid` section is missing. - -[float] -==== `auth.edgegrid.client_token` - -Client Token generated from https://control.akamai.com/ - -[float] -==== `auth.edgegrid.client_secret` - -Client Secret generated from https://control.akamai.com/ - -[float] -==== `auth.edgegrid.access_token` - -Access Token generated from https://control.akamai.com/ - -[float] -==== `auth.edgegrid.host` - -Unique API host generated from https://control.akamai.com/. In the format of `akaa-xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx.luna.akamaiapis.net` -with no path or scheme - [float] ==== `request.url` diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go b/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go index b0ae89c0cd89..0d862ea181b2 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_auth.go @@ -19,27 +19,16 @@ import ( "golang.org/x/oauth2/endpoints" "golang.org/x/oauth2/google" - "github.com/akamai/AkamaiOPEN-edgegrid-golang/edgegrid" "github.com/elastic/beats/v7/libbeat/common" ) type authConfig struct { - Basic *basicAuthConfig `config:"basic"` - OAuth2 *oAuth2Config `config:"oauth2"` - EdgeGrid *edgeGridConfig `config:"edgegrid"` + Basic *basicAuthConfig `config:"basic"` + OAuth2 *oAuth2Config `config:"oauth2"` } func (c authConfig) Validate() error { - list := []bool{c.Basic.isEnabled(), c.OAuth2.isEnabled(), c.EdgeGrid.isEnabled()} - filtered := []bool{} - - for i := range list { - if list[i] { - filtered = append(filtered, list[i]) - } - } - - if len(filtered) > 1 { + if c.Basic.isEnabled() && c.OAuth2.isEnabled() { return errors.New("only one kind of auth can be enabled") } return nil @@ -274,42 +263,3 @@ func (o *oAuth2Config) validateAzureProvider() error { return nil } - -type edgeGridConfig struct { - Enabled *bool `config:"enabled"` - Host string `config:"host"` - ClientToken string `config:"client_token"` - ClientSecret string `config:"client_secret"` - AccessToken string `config:"access_token"` -} - -// IsEnabled returns true if the `enable` field is set to true in the yaml. -func (e *edgeGridConfig) isEnabled() bool { - return e != nil && (e.Enabled == nil || *e.Enabled) -} - -// Validate checks if oauth2 config is valid. -func (e *edgeGridConfig) Validate() error { - if !e.isEnabled() { - return nil - } - - if e.Host == "" || e.ClientToken == "" || e.ClientSecret == "" || e.AccessToken == "" { - return errors.New("All parameters must be set; Host, Client Token, Client Secret, Access Token.") - } - - return nil -} - -func (e *edgeGridConfig) config() edgegrid.Config { - config := edgegrid.Config{ - Host: e.Host, - ClientToken: e.ClientToken, - ClientSecret: e.ClientSecret, - AccessToken: e.AccessToken, - MaxBody: 1024, - HeaderToSign: []string{}, - Debug: false, - } - return config -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go index 681bfb803795..4110430b25e8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_test.go @@ -392,69 +392,3 @@ func TestCursorEntryConfig(t *testing.T) { assert.True(t, conf["entry3"].mustIgnoreEmptyValue()) assert.True(t, conf["entry4"].mustIgnoreEmptyValue()) } - -func TestConfigEdgeGridValidation(t *testing.T) { - cases := []struct { - name string - expectedErr string - input map[string]interface{} - setup func() - teardown func() - }{ - { - name: "can't set edge grid and basic auth together", - expectedErr: "only one kind of auth can be enabled accessing 'auth'", - input: map[string]interface{}{ - "auth.basic.user": "user", - "auth.basic.password": "pass", - "auth.edgegrid": map[string]interface{}{ - "client_token": "asdf", - "host": "localhost", - "client_secret": "asdf", - "access_token": "asdf", - }, - }, - }, - { - name: "Must set all config parameters", - expectedErr: "All parameters must be set; Host, Client Token, Client Secret, Access Token. accessing 'auth.edgegrid'", - input: map[string]interface{}{ - "auth.edgegrid": map[string]interface{}{ - "client_token": "asdf", - "host": "localhost", - "client_secret": "asdf", - }, - }, - }, - } - - for _, c := range cases { - c := c - t.Run(c.name, func(t *testing.T) { - if c.setup != nil { - c.setup() - } - - if c.teardown != nil { - defer c.teardown() - } - - c.input["request.url"] = "localhost" - cfg := common.MustNewConfigFrom(c.input) - conf := defaultConfig() - err := cfg.Unpack(&conf) - - switch { - case c.expectedErr == "": - if err != nil { - t.Fatalf("Configuration validation failed. no error expected but got %q", err) - } - - case c.expectedErr != "": - if err == nil || err.Error() != c.expectedErr { - t.Fatalf("Configuration validation failed. expecting %q error but got %q", c.expectedErr, err) - } - } - }) - } -} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index 8615e5a04503..6ea063a10af0 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -78,9 +78,6 @@ func newPaginationRequestFactory(method, encodeAs string, url url.URL, body *com rf.user = authConfig.Basic.User rf.password = authConfig.Basic.Password } - if authConfig != nil && authConfig.EdgeGrid.isEnabled() { - rf.EdgeGridConfig = authConfig.EdgeGrid.config() - } return rf } @@ -138,7 +135,7 @@ func (iter *pageIterator) next() (*response, bool, error) { return nil, false, err } - resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq, iter.pagination.requestFactory) + resp, err := iter.pagination.httpClient.do(iter.stdCtx, iter.trCtx, httpReq) if err != nil { return nil, false, err } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index 4baafbc06751..921b13b9ab71 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -12,8 +12,6 @@ import ( "net/http" "net/url" - egclient "github.com/akamai/AkamaiOPEN-edgegrid-golang/client-v1" - "github.com/akamai/AkamaiOPEN-edgegrid-golang/edgegrid" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -32,14 +30,9 @@ type httpClient struct { limiter *rateLimiter } -func (c *httpClient) do(stdCtx context.Context, trCtx *transformContext, req *http.Request, rf *requestFactory) (*http.Response, error) { +func (c *httpClient) do(stdCtx context.Context, trCtx *transformContext, req *http.Request) (*http.Response, error) { resp, err := c.limiter.execute(stdCtx, func() (*http.Response, error) { - - if rf.EdgeGridConfig.Host != "" { - return egclient.Do(rf.EdgeGridConfig, req) - } else { - return c.client.Do(req) - } + return c.client.Do(req) }) if err != nil { return nil, fmt.Errorf("failed to execute http client.Do: %w", err) @@ -87,15 +80,14 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro } type requestFactory struct { - url url.URL - method string - body *common.MapStr - transforms []basicTransform - user string - password string - log *logp.Logger - encoder encoderFunc - EdgeGridConfig edgegrid.Config + url url.URL + method string + body *common.MapStr + transforms []basicTransform + user string + password string + log *logp.Logger + encoder encoderFunc } func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { @@ -113,9 +105,6 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. rf.user = authConfig.Basic.User rf.password = authConfig.Basic.Password } - if authConfig != nil && authConfig.EdgeGrid.isEnabled() { - rf.EdgeGridConfig = authConfig.EdgeGrid.config() - } return rf } @@ -139,9 +128,6 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx *transfor url := trReq.url() req, err := http.NewRequest(rf.method, url.String(), bytes.NewBuffer(body)) - if rf.EdgeGridConfig.Host != "" { - req, err = egclient.NewRequest(rf.EdgeGridConfig, rf.method, url.String(), nil) - } if err != nil { return nil, err } @@ -183,7 +169,7 @@ func (r *requester) doRequest(stdCtx context.Context, trCtx *transformContext, p return fmt.Errorf("failed to create http request: %w", err) } - httpResp, err := r.client.do(stdCtx, trCtx, req, r.requestFactory) + httpResp, err := r.client.do(stdCtx, trCtx, req) if err != nil { return fmt.Errorf("failed to execute http client.Do: %w", err) } From 8d1f1f28424324a6a3291e8c4de518a5ba75f394 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 15 Sep 2021 13:06:02 +0000 Subject: [PATCH 5/7] Add UUID and Base64 HMAC template functions --- CHANGELOG.next.asciidoc | 1 + go.mod | 2 +- go.sum | 2 ++ .../docs/inputs/input-httpjson.asciidoc | 4 ++- .../input/httpjson/internal/v2/value_tpl.go | 12 +++++++- .../httpjson/internal/v2/value_tpl_test.go | 30 +++++++++++++++++-- 6 files changed, 45 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 86b1ede2652b..394b6d62a7df 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -757,6 +757,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879] - Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937] - Add Akamai EdgeGrid authenication to `httpjson` input {pull}27873[27873] +- Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873] *Heartbeat* diff --git a/go.mod b/go.mod index 07baf547449f..25c03329e8f9 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/google/flatbuffers v1.12.0 github.com/google/go-cmp v0.5.4 github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.3.0 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/gorilla/mux v1.7.2 github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect diff --git a/go.sum b/go.sum index 793736682864..68cfd3c523eb 100644 --- a/go.sum +++ b/go.sum @@ -405,6 +405,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 0585f10d46e3..4180cab1b071 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -202,11 +202,13 @@ Some built-in helper functions are provided to work with the input state inside - `add`: adds a list of integers and returns their sum. - `mul`: multiplies two integers. - `div`: does the integer division of two integer values. -- `hmac`: calculates the hmac signature of a list of strings concatenated together. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` +- `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` - `base64Encode`: Joins and base64 encodes all supplied strings. Example `[[base64Encode "string1" "string2"]]` - `base64EncodeNoPad`: Joins and base64 encodes all supplied strings without padding. Example `[[base64EncodeNoPad "string1" "string2"]]` - `join`: joins a list of strings using the specified separator. Example: `[[join .body.arr ","]]` - `sprintf`: formats according to a format specifier and returns the resulting string. Refer to https://pkg.go.dev/fmt#Sprintf[the Go docs] for usage. Example: `[[sprintf "%d:%q" 34 "quote this"]]` +- `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` +- `uuid`: returns a random UUID such as `a11e8780-e3e7-46d0-8e76-f66e75acf019` Example: `[[ uuid ]]` In addition to the provided functions, any of the native functions for https://golang.org/pkg/time/#Time[`time.Time`], https://golang.org/pkg/net/http/#Header[`http.Header`], and https://golang.org/pkg/net/url/#Values[`url.Values`] types can be used on the corresponding objects. Examples: `[[(now).Day]]`, `[[.last_response.header.Get "key"]]` diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index ed00cf97bbef..7d779f048442 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -22,6 +22,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/google/uuid" ) // we define custom delimiters to prevent issues when using template values as part of other Go templates. @@ -56,11 +57,12 @@ func (t *valueTpl) Unpack(in string) error { "mul": mul, "div": div, "hmac": hmacStringHex, - "hmacBase64": hmacStringBase64, "base64Encode": base64Encode, "base64EncodeNoPad": base64EncodeNoPad, "join": strings.Join, "sprintf": fmt.Sprintf, + "hmacBase64": hmacStringBase64, + "uuid": uuidString, }). Delims(leftDelim, rightDelim). Parse(in) @@ -310,3 +312,11 @@ func hmacStringBase64(hmacType string, hmacKey string, values ...string) string // Get result and encode as hexadecimal string return base64.StdEncoding.EncodeToString(bytes) } + +func uuidString() string { + uuid, err := uuid.NewRandom() + if err != nil { + return "" + } + return uuid.String() +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 4586dec77112..cd9f37bd48b8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -253,14 +253,14 @@ func TestValueTpl(t *testing.T) { expectedVal: "4", }, { - name: "func sha1 hmac", + name: "func sha1 hmac Hex", value: `[[hmac "sha1" "secret" "string1" "string2"]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, expectedVal: "87eca1e7cba012b2dd4a907c2ad4345a252a38f4", }, { - name: "func sha256 hmac", + name: "func sha256 hmac Hex", setup: func() { timeNow = func() time.Time { return time.Unix(1627697597, 0).UTC() } }, teardown: func() { timeNow = time.Now }, value: `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`, @@ -269,7 +269,7 @@ func TestValueTpl(t *testing.T) { expectedVal: "adc61cd206e146f2d1337504e760ea70f3d2e34bedf28d07802e0e776568a06b", }, { - name: "func invalid hmac", + name: "func invalid hmac Hex", value: `[[hmac "md5" "secret" "string1" "string2"]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, @@ -331,6 +331,30 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: `"foo,bar":1`, }, + { + name: "func sha1 hmac Base64", + value: `[[hmacBase64 "sha1" "secret" "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "h+yh58ugErLdSpB8KtQ0WiUqOPQ=", + }, + { + name: "func sha256 hmac Base64", + setup: func() { timeNow = func() time.Time { return time.Unix(1627697597, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `[[hmacBase64 "sha256" "secret" "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "HlglO6yRZs0Ts3MjmgnRKtTJk3fr9nt8LmeliVKZyAA=", + }, + { + name: "func invalid hmac Base64", + value: `[[hmacBase64 "md5" "secret" "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "", + expectedError: errEmptyTemplateResult.Error(), + }, } for _, tc := range cases { From f1b0d87df99a3281e039da322689653dc110616a Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 17 Sep 2021 11:48:12 +0000 Subject: [PATCH 6/7] don't upgrade google/uuid library --- CHANGELOG.next.asciidoc | 2 -- go.mod | 2 +- go.sum | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 394b6d62a7df..6e23dd7275d4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -756,10 +756,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `ignore_empty_value` flag to `httpjson` `split` processor. {pull}27880[27880] - Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879] - Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937] -- Add Akamai EdgeGrid authenication to `httpjson` input {pull}27873[27873] - Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873] - *Heartbeat* - Add mime type detection for http responses. {pull}22976[22976] diff --git a/go.mod b/go.mod index 25c03329e8f9..07baf547449f 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/google/flatbuffers v1.12.0 github.com/google/go-cmp v0.5.4 github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.1.2 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/gorilla/mux v1.7.2 github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect diff --git a/go.sum b/go.sum index 68cfd3c523eb..c14acc22e90a 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,6 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -758,7 +756,6 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 h1:0gYLpmzecnaDCoeWxSfEJ7J1b6B/67+NV++4HKQXx+Y= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= From c08213abd7d3c7d8abd4b173302c2d602de37944 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Mon, 20 Sep 2021 11:20:17 +0200 Subject: [PATCH 7/7] Format value_tpl.go --- x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index 7d779f048442..81472f4df514 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -21,8 +21,9 @@ import ( "text/template" "time" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/google/uuid" + + "github.com/elastic/beats/v7/libbeat/logp" ) // we define custom delimiters to prevent issues when using template values as part of other Go templates.