From 34f0922988284144c37895a451255d56e94676da Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 28 Oct 2021 13:56:45 -0400 Subject: [PATCH] Allow fingerprint processor to read @timestamp Use `beat.Event.GetValue` so that @timestamp and @metadata are accessible to be hashed. Also clarify the documentation to explain how the value being hashed is constructed. Fixes #28683 --- CHANGELOG.next.asciidoc | 1 + .../fingerprint/docs/fingerprint.asciidoc | 6 +++- libbeat/processors/fingerprint/errors.go | 2 +- libbeat/processors/fingerprint/fingerprint.go | 34 ++++++++++--------- .../fingerprint/fingerprint_test.go | 11 +++++- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d8e8b320023..29efa81fa1d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -229,6 +229,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro - Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280] - Allow `clone3` syscall in seccomp filters. {pull}28117[28117] - Remove unnecessary escaping step in dashboard loading, so they can be displayed in Kibana. {pull}28395[28395] +- Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683] *Auditbeat* diff --git a/libbeat/processors/fingerprint/docs/fingerprint.asciidoc b/libbeat/processors/fingerprint/docs/fingerprint.asciidoc index 75509020c0a..04a3bfa3d95 100644 --- a/libbeat/processors/fingerprint/docs/fingerprint.asciidoc +++ b/libbeat/processors/fingerprint/docs/fingerprint.asciidoc @@ -8,6 +8,9 @@ The `fingerprint` processor generates a fingerprint of an event based on a specified subset of its fields. +The value that is hashed is constructed as a concatenation of the field name and +field value separated by `|`. For example `|field1|value1|field2|value2|`. + [source,yaml] ----------------------------------------------------- processors: @@ -17,7 +20,8 @@ processors: The following settings are supported: -`fields`:: List of fields to use as the source for the fingerprint. +`fields`:: List of fields to use as the source for the fingerprint. The list +will be alphabetically sorted by the processor. `ignore_missing`:: (Optional) Whether to ignore missing fields. Default is `false`. `target_field`:: (Optional) Field in which the generated fingerprint should be stored. Default is `fingerprint`. `method`:: (Optional) Algorithm to use for computing the fingerprint. Must be one of: `md5`, `sha1`, `sha256`, `sha384`, `sha512`, `xxhash`. Default is `sha256`. diff --git a/libbeat/processors/fingerprint/errors.go b/libbeat/processors/fingerprint/errors.go index e025015883b..94baf2984bd 100644 --- a/libbeat/processors/fingerprint/errors.go +++ b/libbeat/processors/fingerprint/errors.go @@ -54,7 +54,7 @@ func makeErrConfigUnpack(cause error) errConfigUnpack { return errConfigUnpack{cause} } func (e errConfigUnpack) Error() string { - return fmt.Sprintf("failed to unpack %v processor configuration: %v", processorName, e.cause) + return fmt.Sprintf("failed to unpack %v processor configuration: %v", procName, e.cause) } func makeErrComputeFingerprint(cause error) errComputeFingerprint { diff --git a/libbeat/processors/fingerprint/fingerprint.go b/libbeat/processors/fingerprint/fingerprint.go index 3028f7a20fe..7637b317572 100644 --- a/libbeat/processors/fingerprint/fingerprint.go +++ b/libbeat/processors/fingerprint/fingerprint.go @@ -18,8 +18,10 @@ package fingerprint import ( + "encoding/json" "fmt" "io" + "strings" "time" "github.com/elastic/beats/v7/libbeat/beat" @@ -28,13 +30,15 @@ import ( jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" ) +const ( + procName = "fingerprint" +) + func init() { - processors.RegisterPlugin("fingerprint", New) - jsprocessor.RegisterPlugin("Fingerprint", New) + processors.RegisterPlugin(procName, New) + jsprocessor.RegisterPlugin(strings.Title(procName), New) } -const processorName = "fingerprint" - type fingerprint struct { config Config fields []string @@ -62,19 +66,17 @@ func New(cfg *common.Config) (processors.Processor, error) { return p, nil } -// Run enriches the given event with fingerprint information +// Run enriches the given event with a fingerprint. func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { hashFn := p.hash() - err := p.writeFields(hashFn, event.Fields) - if err != nil { + if err := p.writeFields(hashFn, event); err != nil { return nil, makeErrComputeFingerprint(err) } - hash := hashFn.Sum(nil) - encodedHash := p.config.Encoding(hash) + encodedHash := p.config.Encoding(hashFn.Sum(nil)) - if _, err = event.PutValue(p.config.TargetField, encodedHash); err != nil { + if _, err := event.PutValue(p.config.TargetField, encodedHash); err != nil { return nil, makeErrComputeFingerprint(err) } @@ -82,12 +84,13 @@ func (p *fingerprint) Run(event *beat.Event) (*beat.Event, error) { } func (p *fingerprint) String() string { - return fmt.Sprintf("%v=[method=[%v]]", processorName, p.config.Method) + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) } -func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error { +func (p *fingerprint) writeFields(to io.Writer, event *beat.Event) error { for _, k := range p.fields { - v, err := eventFields.GetValue(k) + v, err := event.GetValue(k) if err != nil { if p.config.IgnoreMissing { continue @@ -95,16 +98,15 @@ func (p *fingerprint) writeFields(to io.Writer, eventFields common.MapStr) error return makeErrMissingField(k, err) } - i := v switch vv := v.(type) { case map[string]interface{}, []interface{}, common.MapStr: return makeErrNonScalarField(k) case time.Time: // Ensure we consistently hash times in UTC. - i = vv.UTC() + v = vv.UTC() } - fmt.Fprintf(to, "|%v|%v", k, i) + fmt.Fprintf(to, "|%v|%v", k, v) } io.WriteString(to, "|") diff --git a/libbeat/processors/fingerprint/fingerprint_test.go b/libbeat/processors/fingerprint/fingerprint_test.go index 957ab589876..044c57a2eed 100644 --- a/libbeat/processors/fingerprint/fingerprint_test.go +++ b/libbeat/processors/fingerprint/fingerprint_test.go @@ -54,6 +54,15 @@ func TestWithConfig(t *testing.T) { }, want: "14a0364b79acbe4c78dd5e77db2c93ae8c750518b32581927d50b3eef407184e", }, + "with @timestamp": { + config: common.MapStr{ + "fields": []string{"@timestamp", "message"}, + }, + input: common.MapStr{ + "message": `test message "hello world"`, + }, + want: "081da76e049554943843b83948ac83ab7aa79fd2849331813e02042586021c26", + }, } for name, test := range cases { @@ -63,7 +72,7 @@ func TestWithConfig(t *testing.T) { require.NoError(t, err) testEvent := &beat.Event{ - Timestamp: time.Now(), + Timestamp: time.Unix(1635443183, 0), Fields: test.input.Clone(), } newEvent, err := p.Run(testEvent)