From 714b7ef8ede9489bc2447b137d182cdeee115360 Mon Sep 17 00:00:00 2001 From: Evgeniy Belyi Date: Thu, 13 Jul 2023 19:25:20 +0300 Subject: [PATCH 1/4] Decrease Clones (#35945) * Decrease Clones * Adding changelog * CodeReview changes (cherry picked from commit faf88b7d697ae708d1ecaa18214801717f2f014f) # Conflicts: # libbeat/beat/event.go # libbeat/beat/event_test.go --- CHANGELOG.next.asciidoc | 32 ++++++ libbeat/beat/event.go | 63 +++++++++-- libbeat/beat/event_test.go | 226 +++++++++++++++++++++++++++++++------ 3 files changed, 279 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c4222839a9d4..42d7601f8762 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -29,6 +29,38 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d ==== Bugfixes *Affecting all Beats* +- Support for multiline zookeeper logs {issue}2496[2496] +- Allow `clock_nanosleep` in the default seccomp profiles for amd64 and 386. Newer versions of glibc (e.g. 2.31) require it. {issue}33792[33792] +- Disable lockfile when running under elastic-agent. {pull}33988[33988] +- Fix lockfile logic, retry locking {pull}34194[34194] +- Add checks to ensure reloading of units if the configuration actually changed. {pull}34346[34346] +- Fix namespacing on self-monitoring {pull}32336[32336] +- Fix race condition when stopping runners {pull}32433[32433] +- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491] +- Log errors from the Elastic Agent V2 client errors channel. Avoids blocking when error occurs communicating with the Elastic Agent. {pull}34392[34392] +- Only log publish event messages in trace log level under elastic-agent. {pull}34391[34391] +- Fix issue where updating a single Elastic Agent configuration unit results in other units being turned off. {pull}34504[34504] +- Fix dropped events when monitor a beat under the agent and send its `Host info` log entry. {pull}34599[34599] +- Fix namespacing on self-monitoring {pull}32336[32336] +- Fix race condition when stopping runners {pull}32433[32433] +- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491] +- Fix panics when a processor is closed twice {pull}34647[34647] +- Update elastic-agent-system-metrics to v0.4.6 to allow builds on mips platforms. {pull}34674[34674] +- The Elasticsearch output now splits large requests instead of dropping them when it receives a StatusRequestEntityTooLarge error. {pull}34911[34911] +- Fix Beats started by agent do not respect the allow_older_versions: true configuration flag {issue}34227[34227] {pull}34964[34964] +- Fix performance issues when we have a lot of inputs starting and stopping by allowing to disable global processors under fleet. {issue}35000[35000] {pull}35031[35031] +- In cases where the matcher detects a non-string type in a match statement, report the error as a debug statement, and not a warning statement. {pull}35119[35119] +- 'add_cloud_metadata' processor - add cloud.region field for GCE cloud provider +- 'add_cloud_metadata' processor - update azure metadata api version to get missing `cloud.account.id` field +- Make sure k8s watchers are closed when closing k8s meta processor. {pull}35630[35630] +- Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640] +- Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820] +- Do not print context cancelled error message when running under agent {pull}36006[36006] +- Fix recovering from invalid output configuration when running under Elastic-Agent {pull}36016[36016] +- Improve StreamBuf append to improve performance when reading long lines from files. {pull}35928[35928] +- Eliminate cloning of event in deepUpdate {pull}35945[35945] + +*Auditbeat* *Filebeat* diff --git a/libbeat/beat/event.go b/libbeat/beat/event.go index 7fd016a14dc0..9974f8fd50b6 100644 --- a/libbeat/beat/event.go +++ b/libbeat/beat/event.go @@ -98,8 +98,8 @@ func (e *Event) deepUpdate(d common.MapStr, overwrite bool) { if len(d) == 0 { return } - fieldsUpdate := d.Clone() // so we can delete redundant keys +<<<<<<< HEAD var metaUpdate common.MapStr for fieldKey, value := range d { @@ -129,14 +129,63 @@ func (e *Event) deepUpdate(d common.MapStr, overwrite bool) { if e.Meta == nil { e.Meta = common.MapStr{} } +======= + // It's supported to update the timestamp using this function. + // However, we must handle it separately since it's a separate field of the event. + timestampValue, timestampExists := d[timestampFieldKey] + if timestampExists { +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) if overwrite { - e.Meta.DeepUpdate(metaUpdate) - } else { - e.Meta.DeepUpdateNoOverwrite(metaUpdate) + _ = e.setTimestamp(timestampValue) } + + // Temporary delete it from the update map, + // so we can do `e.Fields.DeepUpdate(d)` or + // `e.Fields.DeepUpdateNoOverwrite(d)` later + delete(d, timestampFieldKey) } - if len(fieldsUpdate) == 0 { + // It's supported to update the metadata using this function. + // However, we must handle it separately since it's a separate field of the event. + metaValue, metaExists := d[metadataFieldKey] + if metaExists { + var metaUpdate mapstr.M + + switch meta := metaValue.(type) { + case mapstr.M: + metaUpdate = meta + case map[string]interface{}: + metaUpdate = mapstr.M(meta) + } + + if metaUpdate != nil { + if e.Meta == nil { + e.Meta = mapstr.M{} + } + if overwrite { + e.Meta.DeepUpdate(metaUpdate) + } else { + e.Meta.DeepUpdateNoOverwrite(metaUpdate) + } + } + + // Temporary delete it from the update map, + // so we can do `e.Fields.DeepUpdate(d)` or + // `e.Fields.DeepUpdateNoOverwrite(d)` later + delete(d, metadataFieldKey) + } + + // At the end we revert all changes we made to the update map + defer func() { + if timestampExists { + d[timestampFieldKey] = timestampValue + } + if metaExists { + d[metadataFieldKey] = metaValue + } + }() + + if len(d) == 0 { return } @@ -145,9 +194,9 @@ func (e *Event) deepUpdate(d common.MapStr, overwrite bool) { } if overwrite { - e.Fields.DeepUpdate(fieldsUpdate) + e.Fields.DeepUpdate(d) } else { - e.Fields.DeepUpdateNoOverwrite(fieldsUpdate) + e.Fields.DeepUpdateNoOverwrite(d) } } diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index 2f4f226ecf94..96d6f283e0c7 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -18,6 +18,7 @@ package beat import ( + "crypto/rand" "testing" "time" @@ -26,11 +27,59 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +const ( + propSize = 1024 * 2014 * 10 +) + +var largeProp string + +func init() { + b := make([]byte, propSize) + _, _ = rand.Read(b) + largeProp = string(b) +} + func newEmptyEvent() *Event { return &Event{Fields: common.MapStr{}} } -func TestEventPutGetTimestamp(t *testing.T) { +func newEvent(e mapstr.M) *Event { + n := &mapstr.M{ + "Fields": mapstr.M{ + "large_prop": largeProp, + }, + } + n.DeepUpdate(e) + var ts time.Time + var meta mapstr.M + var fields mapstr.M + var private mapstr.M + + v, ex := (*n)["Timestamp"] + if ex { + ts = v.(time.Time) + } + v, ex = (*n)["Meta"] + if ex { + meta = v.(mapstr.M) + } + v, ex = (*n)["Fields"] + if ex { + fields = v.(mapstr.M) + } + v, ex = (*n)["Private"] + if ex { + private = v.(mapstr.M) + } + return &Event{ + Timestamp: ts, + Meta: meta, + Fields: fields, + Private: private, + } +} + +func BenchmarkTestEventPutGetTimestamp(b *testing.B) { evt := newEmptyEvent() ts := time.Now() @@ -38,17 +87,17 @@ func TestEventPutGetTimestamp(t *testing.T) { v, err := evt.GetValue("@timestamp") if err != nil { - t.Fatal(err) + b.Fatal(err) } - assert.Equal(t, ts, v) - assert.Equal(t, ts, evt.Timestamp) + assert.Equal(b, ts, v) + assert.Equal(b, ts, evt.Timestamp) // The @timestamp is not written into Fields. - assert.Nil(t, evt.Fields["@timestamp"]) + assert.Nil(b, evt.Fields["@timestamp"]) } -func TestDeepUpdate(t *testing.T) { +func BenchmarkTestDeepUpdate(b *testing.B) { ts := time.Now() cases := []struct { @@ -60,6 +109,7 @@ func TestDeepUpdate(t *testing.T) { }{ { name: "does nothing if no update", +<<<<<<< HEAD event: &Event{}, update: common.MapStr{}, expected: &Event{}, @@ -68,31 +118,60 @@ func TestDeepUpdate(t *testing.T) { name: "updates timestamp", event: &Event{}, update: common.MapStr{ +======= + event: newEvent(mapstr.M{}), + update: mapstr.M{}, + expected: newEvent(mapstr.M{}), + }, + { + name: "updates timestamp", + event: newEvent(mapstr.M{}), + update: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) timestampFieldKey: ts, }, overwrite: true, expected: &Event{ Timestamp: ts, + Fields: mapstr.M{ + "large_prop": largeProp, + }, }, }, { name: "does not overwrite timestamp", +<<<<<<< HEAD event: &Event{ Timestamp: ts, }, update: common.MapStr{ +======= + event: newEvent(mapstr.M{ + "Timestamp": ts, + }), + update: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) timestampFieldKey: time.Now().Add(time.Hour), }, overwrite: false, expected: &Event{ Timestamp: ts, + Fields: mapstr.M{ + "large_prop": largeProp, + }, }, }, { name: "initializes metadata if nil", +<<<<<<< HEAD event: &Event{}, update: common.MapStr{ metadataFieldKey: common.MapStr{ +======= + event: newEvent(mapstr.M{}), + update: mapstr.M{ + metadataFieldKey: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, @@ -102,10 +181,14 @@ func TestDeepUpdate(t *testing.T) { "first": "new", "second": 42, }, + Fields: mapstr.M{ + "large_prop": largeProp, + }, }, }, { name: "updates metadata but does not overwrite", +<<<<<<< HEAD event: &Event{ Meta: common.MapStr{ "first": "initial", @@ -113,6 +196,15 @@ func TestDeepUpdate(t *testing.T) { }, update: common.MapStr{ metadataFieldKey: common.MapStr{ +======= + event: newEvent(mapstr.M{ + "Meta": mapstr.M{ + "first": "initial", + }, + }), + update: mapstr.M{ + metadataFieldKey: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, @@ -123,10 +215,14 @@ func TestDeepUpdate(t *testing.T) { "first": "initial", "second": 42, }, + Fields: mapstr.M{ + "large_prop": largeProp, + }, }, }, { name: "updates metadata and overwrites", +<<<<<<< HEAD event: &Event{ Meta: common.MapStr{ "first": "initial", @@ -134,6 +230,15 @@ func TestDeepUpdate(t *testing.T) { }, update: common.MapStr{ metadataFieldKey: common.MapStr{ +======= + event: newEvent(mapstr.M{ + "Meta": mapstr.M{ + "first": "initial", + }, + }), + update: mapstr.M{ + metadataFieldKey: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, @@ -144,158 +249,209 @@ func TestDeepUpdate(t *testing.T) { "first": "new", "second": 42, }, + Fields: mapstr.M{ + "large_prop": largeProp, + }, }, }, { name: "updates fields but does not overwrite", +<<<<<<< HEAD event: &Event{ Fields: common.MapStr{ "first": "initial", }, }, update: common.MapStr{ +======= + event: newEvent(mapstr.M{ + "Fields": mapstr.M{ + "first": "initial", + }, + }), + update: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, overwrite: false, expected: &Event{ +<<<<<<< HEAD Fields: common.MapStr{ "first": "initial", "second": 42, +======= + Fields: mapstr.M{ + "first": "initial", + "second": 42, + "large_prop": largeProp, +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) }, }, }, { name: "updates metadata and overwrites", +<<<<<<< HEAD event: &Event{ Fields: common.MapStr{ "first": "initial", }, }, update: common.MapStr{ +======= + event: newEvent(mapstr.M{ + "Fields": mapstr.M{ + "first": "initial", + }, + }), + update: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, overwrite: true, expected: &Event{ +<<<<<<< HEAD Fields: common.MapStr{ "first": "new", "second": 42, +======= + Fields: mapstr.M{ + "first": "new", + "second": 42, + "large_prop": largeProp, +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) }, }, }, { name: "initializes fields if nil", +<<<<<<< HEAD event: &Event{}, update: common.MapStr{ +======= + event: newEvent(mapstr.M{}), + update: mapstr.M{ +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, expected: &Event{ +<<<<<<< HEAD Fields: common.MapStr{ "first": "new", "second": 42, +======= + Fields: mapstr.M{ + "first": "new", + "second": 42, + "large_prop": largeProp, +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) }, }, }, } for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { + b.Run(tc.name, func(b *testing.B) { tc.event.deepUpdate(tc.update, tc.overwrite) - assert.Equal(t, tc.expected.Timestamp, tc.event.Timestamp) - assert.Equal(t, tc.expected.Fields, tc.event.Fields) - assert.Equal(t, tc.expected.Meta, tc.event.Meta) + assert.Equal(b, tc.expected.Timestamp, tc.event.Timestamp) + assert.Equal(b, tc.expected.Fields, tc.event.Fields) + assert.Equal(b, tc.expected.Meta, tc.event.Meta) }) } } -func TestEventMetadata(t *testing.T) { +func BenchmarkTestEventMetadata(b *testing.B) { const id = "123" newMeta := func() common.MapStr { return common.MapStr{"_id": id} } - t.Run("put", func(t *testing.T) { + b.Run("put", func(b *testing.B) { evt := newEmptyEvent() meta := newMeta() evt.PutValue("@metadata", meta) - assert.Equal(t, meta, evt.Meta) - assert.Empty(t, evt.Fields) + assert.Equal(b, meta, evt.Meta) + assert.Empty(b, evt.Fields) }) - t.Run("get", func(t *testing.T) { + b.Run("get", func(b *testing.B) { evt := newEmptyEvent() evt.Meta = newMeta() meta, err := evt.GetValue("@metadata") - assert.NoError(t, err) - assert.Equal(t, evt.Meta, meta) + assert.NoError(b, err) + assert.Equal(b, evt.Meta, meta) }) - t.Run("put sub-key", func(t *testing.T) { + b.Run("put sub-key", func(b *testing.B) { evt := newEmptyEvent() evt.PutValue("@metadata._id", id) - assert.Equal(t, newMeta(), evt.Meta) - assert.Empty(t, evt.Fields) + assert.Equal(b, newMeta(), evt.Meta) + assert.Empty(b, evt.Fields) }) - t.Run("get sub-key", func(t *testing.T) { + b.Run("get sub-key", func(b *testing.B) { evt := newEmptyEvent() evt.Meta = newMeta() v, err := evt.GetValue("@metadata._id") - assert.NoError(t, err) - assert.Equal(t, id, v) + assert.NoError(b, err) + assert.Equal(b, id, v) }) - t.Run("delete", func(t *testing.T) { + b.Run("delete", func(b *testing.B) { evt := newEmptyEvent() evt.Meta = newMeta() err := evt.Delete("@metadata") - assert.NoError(t, err) - assert.Nil(t, evt.Meta) + assert.NoError(b, err) + assert.Nil(b, evt.Meta) }) - t.Run("delete sub-key", func(t *testing.T) { + b.Run("delete sub-key", func(b *testing.B) { evt := newEmptyEvent() evt.Meta = newMeta() err := evt.Delete("@metadata._id") - assert.NoError(t, err) - assert.Empty(t, evt.Meta) + assert.NoError(b, err) + assert.Empty(b, evt.Meta) }) - t.Run("setID", func(t *testing.T) { + b.Run("setID", func(b *testing.B) { evt := newEmptyEvent() evt.SetID(id) - assert.Equal(t, newMeta(), evt.Meta) + assert.Equal(b, newMeta(), evt.Meta) }) - t.Run("put non-metadata", func(t *testing.T) { + b.Run("put non-metadata", func(b *testing.B) { evt := newEmptyEvent() evt.PutValue("@metadataSpecial", id) +<<<<<<< HEAD assert.Equal(t, common.MapStr{"@metadataSpecial": id}, evt.Fields) +======= + assert.Equal(b, mapstr.M{"@metadataSpecial": id}, evt.Fields) +>>>>>>> faf88b7d69 (Decrease Clones (#35945)) }) - t.Run("delete non-metadata", func(t *testing.T) { + b.Run("delete non-metadata", func(b *testing.B) { evt := newEmptyEvent() evt.Meta = newMeta() err := evt.Delete("@metadataSpecial") - assert.Error(t, err) - assert.Equal(t, newMeta(), evt.Meta) + assert.Error(b, err) + assert.Equal(b, newMeta(), evt.Meta) }) } From ffda7cdfa0712b46cd34ec254ad7cdb1e9e0e668 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Mon, 18 Sep 2023 13:39:55 +0200 Subject: [PATCH 2/4] Resolve conflicts --- CHANGELOG.next.asciidoc | 40 +--------- libbeat/beat/event.go | 40 +--------- libbeat/beat/event_test.go | 160 ++++++++----------------------------- 3 files changed, 40 insertions(+), 200 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 42d7601f8762..abe22daabcd1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -29,40 +29,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d ==== Bugfixes *Affecting all Beats* -- Support for multiline zookeeper logs {issue}2496[2496] -- Allow `clock_nanosleep` in the default seccomp profiles for amd64 and 386. Newer versions of glibc (e.g. 2.31) require it. {issue}33792[33792] -- Disable lockfile when running under elastic-agent. {pull}33988[33988] -- Fix lockfile logic, retry locking {pull}34194[34194] -- Add checks to ensure reloading of units if the configuration actually changed. {pull}34346[34346] -- Fix namespacing on self-monitoring {pull}32336[32336] -- Fix race condition when stopping runners {pull}32433[32433] -- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491] -- Log errors from the Elastic Agent V2 client errors channel. Avoids blocking when error occurs communicating with the Elastic Agent. {pull}34392[34392] -- Only log publish event messages in trace log level under elastic-agent. {pull}34391[34391] -- Fix issue where updating a single Elastic Agent configuration unit results in other units being turned off. {pull}34504[34504] -- Fix dropped events when monitor a beat under the agent and send its `Host info` log entry. {pull}34599[34599] -- Fix namespacing on self-monitoring {pull}32336[32336] -- Fix race condition when stopping runners {pull}32433[32433] -- Fix concurrent map writes when system/process code called from reporter code {pull}32491[32491] -- Fix panics when a processor is closed twice {pull}34647[34647] -- Update elastic-agent-system-metrics to v0.4.6 to allow builds on mips platforms. {pull}34674[34674] -- The Elasticsearch output now splits large requests instead of dropping them when it receives a StatusRequestEntityTooLarge error. {pull}34911[34911] -- Fix Beats started by agent do not respect the allow_older_versions: true configuration flag {issue}34227[34227] {pull}34964[34964] -- Fix performance issues when we have a lot of inputs starting and stopping by allowing to disable global processors under fleet. {issue}35000[35000] {pull}35031[35031] -- In cases where the matcher detects a non-string type in a match statement, report the error as a debug statement, and not a warning statement. {pull}35119[35119] -- 'add_cloud_metadata' processor - add cloud.region field for GCE cloud provider -- 'add_cloud_metadata' processor - update azure metadata api version to get missing `cloud.account.id` field -- Make sure k8s watchers are closed when closing k8s meta processor. {pull}35630[35630] -- Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640] -- Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820] -- Do not print context cancelled error message when running under agent {pull}36006[36006] -- Fix recovering from invalid output configuration when running under Elastic-Agent {pull}36016[36016] -- Improve StreamBuf append to improve performance when reading long lines from files. {pull}35928[35928] -- Eliminate cloning of event in deepUpdate {pull}35945[35945] - -*Auditbeat* -*Filebeat* +- Eliminate cloning of event in deepUpdate {pull}35945[35945] *Auditbeat* @@ -173,9 +141,3 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* ==== Known Issue - - - - - - diff --git a/libbeat/beat/event.go b/libbeat/beat/event.go index 9974f8fd50b6..68b015dbd460 100644 --- a/libbeat/beat/event.go +++ b/libbeat/beat/event.go @@ -99,42 +99,10 @@ func (e *Event) deepUpdate(d common.MapStr, overwrite bool) { return } -<<<<<<< HEAD - var metaUpdate common.MapStr - - for fieldKey, value := range d { - switch fieldKey { - - // one of the updates is the timestamp which is not a part of the event fields - case timestampFieldKey: - if overwrite { - _ = e.setTimestamp(value) - } - delete(fieldsUpdate, fieldKey) - - // some updates are addressed for the metadata not the fields - case metadataFieldKey: - switch meta := value.(type) { - case common.MapStr: - metaUpdate = meta - case map[string]interface{}: - metaUpdate = common.MapStr(meta) - } - - delete(fieldsUpdate, fieldKey) - } - } - - if metaUpdate != nil { - if e.Meta == nil { - e.Meta = common.MapStr{} - } -======= // It's supported to update the timestamp using this function. // However, we must handle it separately since it's a separate field of the event. timestampValue, timestampExists := d[timestampFieldKey] if timestampExists { ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) if overwrite { _ = e.setTimestamp(timestampValue) } @@ -149,18 +117,18 @@ func (e *Event) deepUpdate(d common.MapStr, overwrite bool) { // However, we must handle it separately since it's a separate field of the event. metaValue, metaExists := d[metadataFieldKey] if metaExists { - var metaUpdate mapstr.M + var metaUpdate common.MapStr switch meta := metaValue.(type) { - case mapstr.M: + case common.MapStr: metaUpdate = meta case map[string]interface{}: - metaUpdate = mapstr.M(meta) + metaUpdate = common.MapStr(meta) } if metaUpdate != nil { if e.Meta == nil { - e.Meta = mapstr.M{} + e.Meta = common.MapStr{} } if overwrite { e.Meta.DeepUpdate(metaUpdate) diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index 96d6f283e0c7..b8e87b8c84a6 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -22,9 +22,8 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" ) const ( @@ -43,17 +42,17 @@ func newEmptyEvent() *Event { return &Event{Fields: common.MapStr{}} } -func newEvent(e mapstr.M) *Event { - n := &mapstr.M{ - "Fields": mapstr.M{ +func newEvent(e common.MapStr) *Event { + n := &common.MapStr{ + "Fields": common.MapStr{ "large_prop": largeProp, }, } n.DeepUpdate(e) var ts time.Time - var meta mapstr.M - var fields mapstr.M - var private mapstr.M + var meta common.MapStr + var fields common.MapStr + var private common.MapStr v, ex := (*n)["Timestamp"] if ex { @@ -61,15 +60,15 @@ func newEvent(e mapstr.M) *Event { } v, ex = (*n)["Meta"] if ex { - meta = v.(mapstr.M) + meta = v.(common.MapStr) } v, ex = (*n)["Fields"] if ex { - fields = v.(mapstr.M) + fields = v.(common.MapStr) } v, ex = (*n)["Private"] if ex { - private = v.(mapstr.M) + private = v.(common.MapStr) } return &Event{ Timestamp: ts, @@ -109,69 +108,45 @@ func BenchmarkTestDeepUpdate(b *testing.B) { }{ { name: "does nothing if no update", -<<<<<<< HEAD - event: &Event{}, + event: newEvent(common.MapStr{}), update: common.MapStr{}, - expected: &Event{}, + expected: newEvent(common.MapStr{}), }, { name: "updates timestamp", - event: &Event{}, + event: newEvent(common.MapStr{}), update: common.MapStr{ -======= - event: newEvent(mapstr.M{}), - update: mapstr.M{}, - expected: newEvent(mapstr.M{}), - }, - { - name: "updates timestamp", - event: newEvent(mapstr.M{}), - update: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) timestampFieldKey: ts, }, overwrite: true, expected: &Event{ Timestamp: ts, - Fields: mapstr.M{ + Fields: common.MapStr{ "large_prop": largeProp, }, }, }, { name: "does not overwrite timestamp", -<<<<<<< HEAD - event: &Event{ - Timestamp: ts, - }, - update: common.MapStr{ -======= - event: newEvent(mapstr.M{ + event: newEvent(common.MapStr{ "Timestamp": ts, }), - update: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) + update: common.MapStr{ timestampFieldKey: time.Now().Add(time.Hour), }, overwrite: false, expected: &Event{ Timestamp: ts, - Fields: mapstr.M{ + Fields: common.MapStr{ "large_prop": largeProp, }, }, }, { name: "initializes metadata if nil", -<<<<<<< HEAD - event: &Event{}, + event: newEvent(common.MapStr{}), update: common.MapStr{ metadataFieldKey: common.MapStr{ -======= - event: newEvent(mapstr.M{}), - update: mapstr.M{ - metadataFieldKey: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, @@ -181,30 +156,20 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "first": "new", "second": 42, }, - Fields: mapstr.M{ + Fields: common.MapStr{ "large_prop": largeProp, }, }, }, { name: "updates metadata but does not overwrite", -<<<<<<< HEAD - event: &Event{ - Meta: common.MapStr{ + event: newEvent(common.MapStr{ + "Meta": common.MapStr{ "first": "initial", }, - }, + }), update: common.MapStr{ metadataFieldKey: common.MapStr{ -======= - event: newEvent(mapstr.M{ - "Meta": mapstr.M{ - "first": "initial", - }, - }), - update: mapstr.M{ - metadataFieldKey: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, @@ -215,30 +180,20 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "first": "initial", "second": 42, }, - Fields: mapstr.M{ + Fields: common.MapStr{ "large_prop": largeProp, }, }, }, { name: "updates metadata and overwrites", -<<<<<<< HEAD - event: &Event{ - Meta: common.MapStr{ + event: newEvent(common.MapStr{ + "Meta": common.MapStr{ "first": "initial", }, - }, + }), update: common.MapStr{ metadataFieldKey: common.MapStr{ -======= - event: newEvent(mapstr.M{ - "Meta": mapstr.M{ - "first": "initial", - }, - }), - update: mapstr.M{ - metadataFieldKey: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, @@ -249,104 +204,63 @@ func BenchmarkTestDeepUpdate(b *testing.B) { "first": "new", "second": 42, }, - Fields: mapstr.M{ + Fields: common.MapStr{ "large_prop": largeProp, }, }, }, { name: "updates fields but does not overwrite", -<<<<<<< HEAD - event: &Event{ - Fields: common.MapStr{ - "first": "initial", - }, - }, - update: common.MapStr{ -======= - event: newEvent(mapstr.M{ - "Fields": mapstr.M{ + event: newEvent(common.MapStr{ + "Fields": common.MapStr{ "first": "initial", }, }), - update: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) + update: common.MapStr{ "first": "new", "second": 42, }, overwrite: false, expected: &Event{ -<<<<<<< HEAD Fields: common.MapStr{ - "first": "initial", - "second": 42, -======= - Fields: mapstr.M{ "first": "initial", "second": 42, "large_prop": largeProp, ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) }, }, }, { name: "updates metadata and overwrites", -<<<<<<< HEAD - event: &Event{ - Fields: common.MapStr{ - "first": "initial", - }, - }, - update: common.MapStr{ -======= - event: newEvent(mapstr.M{ - "Fields": mapstr.M{ + event: newEvent(common.MapStr{ + "Fields": common.MapStr{ "first": "initial", }, }), - update: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) + update: common.MapStr{ "first": "new", "second": 42, }, overwrite: true, expected: &Event{ -<<<<<<< HEAD Fields: common.MapStr{ - "first": "new", - "second": 42, -======= - Fields: mapstr.M{ "first": "new", "second": 42, "large_prop": largeProp, ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) }, }, }, { name: "initializes fields if nil", -<<<<<<< HEAD - event: &Event{}, + event: newEvent(common.MapStr{}), update: common.MapStr{ -======= - event: newEvent(mapstr.M{}), - update: mapstr.M{ ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) "first": "new", "second": 42, }, expected: &Event{ -<<<<<<< HEAD Fields: common.MapStr{ - "first": "new", - "second": 42, -======= - Fields: mapstr.M{ "first": "new", "second": 42, "large_prop": largeProp, ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) }, }, }, @@ -438,11 +352,7 @@ func BenchmarkTestEventMetadata(b *testing.B) { evt.PutValue("@metadataSpecial", id) -<<<<<<< HEAD - assert.Equal(t, common.MapStr{"@metadataSpecial": id}, evt.Fields) -======= - assert.Equal(b, mapstr.M{"@metadataSpecial": id}, evt.Fields) ->>>>>>> faf88b7d69 (Decrease Clones (#35945)) + assert.Equal(b, common.MapStr{"@metadataSpecial": id}, evt.Fields) }) b.Run("delete non-metadata", func(b *testing.B) { From d1872ce747e0dba76a563f86281f52610ac3dab2 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Mon, 18 Sep 2023 14:14:05 +0200 Subject: [PATCH 3/4] Fix linter issues --- libbeat/beat/event_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index b8e87b8c84a6..aacb347df8d7 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -22,8 +22,9 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common" "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" ) const ( @@ -82,7 +83,7 @@ func BenchmarkTestEventPutGetTimestamp(b *testing.B) { evt := newEmptyEvent() ts := time.Now() - evt.PutValue("@timestamp", ts) + _, _ = evt.PutValue("@timestamp", ts) v, err := evt.GetValue("@timestamp") if err != nil { @@ -284,7 +285,7 @@ func BenchmarkTestEventMetadata(b *testing.B) { evt := newEmptyEvent() meta := newMeta() - evt.PutValue("@metadata", meta) + _, _ = evt.PutValue("@metadata", meta) assert.Equal(b, meta, evt.Meta) assert.Empty(b, evt.Fields) @@ -303,7 +304,7 @@ func BenchmarkTestEventMetadata(b *testing.B) { b.Run("put sub-key", func(b *testing.B) { evt := newEmptyEvent() - evt.PutValue("@metadata._id", id) + _, _ = evt.PutValue("@metadata._id", id) assert.Equal(b, newMeta(), evt.Meta) assert.Empty(b, evt.Fields) From 8bb3d5543bf714e6daa0651115b88674a76e82f2 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Mon, 18 Sep 2023 14:26:13 +0200 Subject: [PATCH 4/4] Another linter fix --- libbeat/beat/event_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index aacb347df8d7..1e03268a4a73 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -351,7 +351,7 @@ func BenchmarkTestEventMetadata(b *testing.B) { b.Run("put non-metadata", func(b *testing.B) { evt := newEmptyEvent() - evt.PutValue("@metadataSpecial", id) + _, _ = evt.PutValue("@metadataSpecial", id) assert.Equal(b, common.MapStr{"@metadataSpecial": id}, evt.Fields) })