diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0d60df49c5ae..6e23dd7275d4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -756,6 +756,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `ignore_empty_value` flag to `httpjson` `split` processor. {pull}27880[27880] - Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879] - Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937] +- Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873] *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 0585f10d46e3..4180cab1b071 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -202,11 +202,13 @@ Some built-in helper functions are provided to work with the input state inside - `add`: adds a list of integers and returns their sum. - `mul`: multiplies two integers. - `div`: does the integer division of two integer values. -- `hmac`: calculates the hmac signature of a list of strings concatenated together. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` +- `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` - `base64Encode`: Joins and base64 encodes all supplied strings. Example `[[base64Encode "string1" "string2"]]` - `base64EncodeNoPad`: Joins and base64 encodes all supplied strings without padding. Example `[[base64EncodeNoPad "string1" "string2"]]` - `join`: joins a list of strings using the specified separator. Example: `[[join .body.arr ","]]` - `sprintf`: formats according to a format specifier and returns the resulting string. Refer to https://pkg.go.dev/fmt#Sprintf[the Go docs] for usage. Example: `[[sprintf "%d:%q" 34 "quote this"]]` +- `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]` +- `uuid`: returns a random UUID such as `a11e8780-e3e7-46d0-8e76-f66e75acf019` Example: `[[ uuid ]]` In addition to the provided functions, any of the native functions for https://golang.org/pkg/time/#Time[`time.Time`], https://golang.org/pkg/net/http/#Header[`http.Header`], and https://golang.org/pkg/net/url/#Values[`url.Values`] types can be used on the corresponding objects. Examples: `[[(now).Day]]`, `[[.last_response.header.Get "key"]]` diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go index 9f91fcdccce5..81472f4df514 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go @@ -21,6 +21,8 @@ import ( "text/template" "time" + "github.com/google/uuid" + "github.com/elastic/beats/v7/libbeat/logp" ) @@ -55,11 +57,13 @@ func (t *valueTpl) Unpack(in string) error { "add": add, "mul": mul, "div": div, - "hmac": hmacString, + "hmac": hmacStringHex, "base64Encode": base64Encode, "base64EncodeNoPad": base64EncodeNoPad, "join": strings.Join, "sprintf": fmt.Sprintf, + "hmacBase64": hmacStringBase64, + "uuid": uuidString, }). Delims(leftDelim, rightDelim). Parse(in) @@ -267,10 +271,9 @@ func base64EncodeNoPad(values ...string) string { return base64.RawStdEncoding.EncodeToString([]byte(data)) } -func hmacString(hmacType string, hmacKey string, values ...string) string { - data := strings.Join(values[:], "") +func hmacString(hmacType string, hmacKey string, data string) []byte { if data == "" { - return "" + return nil } // Create a new HMAC by defining the hash type and the key (as byte array) var mac hash.Hash @@ -281,11 +284,40 @@ func hmacString(hmacType string, hmacKey string, values ...string) string { mac = hmac.New(sha1.New, []byte(hmacKey)) default: // Upstream config validation prevents this from happening. - return "" + return nil } // Write Data to it mac.Write([]byte(data)) + // Get result and encode as bytes + return mac.Sum(nil) +} + +func hmacStringHex(hmacType string, hmacKey string, values ...string) string { + data := strings.Join(values[:], "") + if data == "" { + return "" + } + bytes := hmacString(hmacType, hmacKey, data) // Get result and encode as hexadecimal string - return hex.EncodeToString(mac.Sum(nil)) + return hex.EncodeToString(bytes) +} + +func hmacStringBase64(hmacType string, hmacKey string, values ...string) string { + data := strings.Join(values[:], "") + if data == "" { + return "" + } + bytes := hmacString(hmacType, hmacKey, data) + + // Get result and encode as hexadecimal string + return base64.StdEncoding.EncodeToString(bytes) +} + +func uuidString() string { + uuid, err := uuid.NewRandom() + if err != nil { + return "" + } + return uuid.String() } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go index 4586dec77112..cd9f37bd48b8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go @@ -253,14 +253,14 @@ func TestValueTpl(t *testing.T) { expectedVal: "4", }, { - name: "func sha1 hmac", + name: "func sha1 hmac Hex", value: `[[hmac "sha1" "secret" "string1" "string2"]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, expectedVal: "87eca1e7cba012b2dd4a907c2ad4345a252a38f4", }, { - name: "func sha256 hmac", + name: "func sha256 hmac Hex", setup: func() { timeNow = func() time.Time { return time.Unix(1627697597, 0).UTC() } }, teardown: func() { timeNow = time.Now }, value: `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`, @@ -269,7 +269,7 @@ func TestValueTpl(t *testing.T) { expectedVal: "adc61cd206e146f2d1337504e760ea70f3d2e34bedf28d07802e0e776568a06b", }, { - name: "func invalid hmac", + name: "func invalid hmac Hex", value: `[[hmac "md5" "secret" "string1" "string2"]]`, paramCtx: emptyTransformContext(), paramTr: transformable{}, @@ -331,6 +331,30 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: `"foo,bar":1`, }, + { + name: "func sha1 hmac Base64", + value: `[[hmacBase64 "sha1" "secret" "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "h+yh58ugErLdSpB8KtQ0WiUqOPQ=", + }, + { + name: "func sha256 hmac Base64", + setup: func() { timeNow = func() time.Time { return time.Unix(1627697597, 0).UTC() } }, + teardown: func() { timeNow = time.Now }, + value: `[[hmacBase64 "sha256" "secret" "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "HlglO6yRZs0Ts3MjmgnRKtTJk3fr9nt8LmeliVKZyAA=", + }, + { + name: "func invalid hmac Base64", + value: `[[hmacBase64 "md5" "secret" "string1" "string2"]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "", + expectedError: errEmptyTemplateResult.Error(), + }, } for _, tc := range cases {