diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 91b3c4acb6f..f3b44fed8bb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -245,7 +245,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add process.entity_id, process.group.name and process.group.id in add_process_metadata processor. Make fim module with kprobes backend to always add an appropriately configured add_process_metadata processor to enrich file events {pull}38776[38776] *Auditbeat* - +- Add `lowercase` processor. {issue}22254[22254] {pull}41424[41424] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 76189f17cce..f33fb7667c4 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13443,11 +13443,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.12.1 +Version: v0.17.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.17.1/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 3e2fe304b67..e3178b6f5d0 100644 --- a/go.mod +++ b/go.mod @@ -191,7 +191,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.9.0 - github.com/elastic/elastic-agent-libs v0.12.1 + github.com/elastic/elastic-agent-libs v0.17.1 github.com/elastic/elastic-agent-system-metrics v0.11.1 github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/elastic/go-quark v0.2.0 diff --git a/go.sum b/go.sum index ba2722f5baa..93bce761422 100644 --- a/go.sum +++ b/go.sum @@ -342,8 +342,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4= github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE= github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0= -github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= +github.com/elastic/elastic-agent-libs v0.17.1 h1:1MXoc1eHGE8hCdVJ9+qiGiZAGeHzT2QBVVzD/oxwqeU= +github.com/elastic/elastic-agent-libs v0.17.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= github.com/elastic/elastic-agent-system-metrics v0.11.1 h1:BxViQHnqxvvi/65rj3mGwG6Eto6ldFCTnuDTUJnakaU= github.com/elastic/elastic-agent-system-metrics v0.11.1/go.mod h1:3QiMu9wTKJFvpCN+5klgGqasTMNKJbgY3xcoN1KQXJk= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= diff --git a/libbeat/processors/actions/alterFieldProcessor.go b/libbeat/processors/actions/alterFieldProcessor.go new file mode 100644 index 00000000000..8be639b8fee --- /dev/null +++ b/libbeat/processors/actions/alterFieldProcessor.go @@ -0,0 +1,135 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "errors" + "fmt" + "strings" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type alterFieldProcessor struct { + Fields []string + IgnoreMissing bool + FailOnError bool + AlterFullField bool + + processorName string + alterFunc mapstr.AlterFunc +} + +// NewAlterFieldProcessor is an umbrella method for processing events based on provided fields. Such as converting event keys to uppercase/lowercase +func NewAlterFieldProcessor(c *conf.C, processorName string, alterFunc mapstr.AlterFunc) (beat.Processor, error) { + config := struct { + Fields []string `config:"fields"` + IgnoreMissing bool `config:"ignore_missing"` + FailOnError bool `config:"fail_on_error"` + AlterFullField bool `config:"alter_full_field"` + }{ + IgnoreMissing: false, + FailOnError: true, + AlterFullField: true, + } + + if err := c.Unpack(&config); err != nil { + return nil, fmt.Errorf("failed to unpack the %s fields configuration: %w", processorName, err) + } + + // Skip mandatory fields + var configFields []string + var lowerField string + for _, readOnly := range processors.MandatoryExportedFields { + readOnly = strings.ToLower(readOnly) + for _, field := range config.Fields { + // Skip fields that match "readOnly" or start with "readOnly." + lowerField = strings.ToLower(field) + if strings.HasPrefix(lowerField, readOnly+".") || lowerField == readOnly { + continue + } + // Add fields that do not match "readOnly" criteria + configFields = append(configFields, field) + } + } + return &alterFieldProcessor{ + Fields: configFields, + IgnoreMissing: config.IgnoreMissing, + FailOnError: config.FailOnError, + processorName: processorName, + AlterFullField: config.AlterFullField, + alterFunc: alterFunc, + }, nil + +} + +func (a *alterFieldProcessor) String() string { + return fmt.Sprintf("%s fields=%+v", a.processorName, *a) +} + +func (a *alterFieldProcessor) Run(event *beat.Event) (*beat.Event, error) { + var backup *beat.Event + if a.FailOnError { + backup = event.Clone() + } + + for _, field := range a.Fields { + err := a.alter(event, field) + if err != nil { + if a.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { + continue + } + if a.FailOnError { + event = backup + _, _ = event.PutValue("error.message", err.Error()) + return event, err + } + } + } + + return event, nil +} + +func (a *alterFieldProcessor) alter(event *beat.Event, field string) error { + + // modify all segments of the key + if a.AlterFullField { + err := event.Fields.AlterPath(field, mapstr.CaseInsensitiveMode, a.alterFunc) + if err != nil { + return err + } + } else { + // modify only the last segment + segmentCount := strings.Count(field, ".") + err := event.Fields.AlterPath(field, mapstr.CaseInsensitiveMode, func(key string) (string, error) { + if segmentCount > 0 { + segmentCount-- + return key, nil + } + return a.alterFunc(key) + }) + if err != nil { + return err + } + } + + return nil +} diff --git a/libbeat/processors/actions/docs/lowercase.asciidoc b/libbeat/processors/actions/docs/lowercase.asciidoc new file mode 100644 index 00000000000..bdb31cf3e96 --- /dev/null +++ b/libbeat/processors/actions/docs/lowercase.asciidoc @@ -0,0 +1,114 @@ +[[lowercase]] +=== Lowercase fields in events + +++++ +lowercase +++++ + +The `lowercase` processor specifies a list of fields that should be converted to lowercase. This transformation applies to keys that match the specified fields. Matching is performed case-insensitively. + + +==== Examples: + +1. Default scenario + +[source,yaml] +---- +processors: + - rename: + fields: + - "ab.cd" + ignore_missing: false + fail_on_error: true + full_path: true +---- +[source,json] +---- +// Input +{ + "AB": {"CD":"data"}, + "CD": {"ef":"data"} +} + + +// output +{ + "ab": {"cd":"data"}, // `AB.CD` -> `ab.cd` + "CD": {"ef":"data"} +} +---- + +[start=2] +2. When `full_path` is false + +[source,yaml] +---- +processors: + - rename: + fields: + - "ab.cd" + ignore_missing: false + fail_on_error: true + alter_full_field: false +---- + +[source,json] +---- +// Input +{ + "AB": {"CD":"data"}, + "CD": {"ef":"data"} +} + + +// output +{ + "AB": {"cd":"data"}, // `AB.CD` -> `AB.cd` (only `cd` is lowercased) + "CD": {"ef":"data"} +} +---- + +[start=2] +2. In case of non unique path to the key + +[source,yaml] +---- +processors: + - rename: + fields: + - "ab" + ignore_missing: false + fail_on_error: true + alter_full_field: true +---- + +[source,json] +---- +// Input +{ + "ab": "first", + "aB": "second" +} + +// Output +{ + "ab": "first", + "aB": "second", + "err": "... Error: key collision" +} +---- + +==== Configuration: + +The `lowercase` processor has the following configuration settings: + +`fields`:: The field names to lowercase. The match is case-insensitive, e.g. `a.b.c.d` would match `A.b.C.d` or `A.B.C.D`. +`ignore_missing`:: (Optional) Indicates whether to ignore events that lack the source field. + The default is `false`, which will fail processing of an event if a field is missing. +`fail_on_error`:: (Optional) If set to `true` and an error occurs, the changes are reverted and the original event is returned. + If set to `false`, processing continues if an error occurs. Default is `true`. +`alter_full_field`:: (Optional) If set to `true`, the entire key path is lowercased. If set to `false` only the final part of the key path is lowercased. Default is true + + + +See <> for a list of supported conditions. diff --git a/libbeat/processors/actions/lowercase.go b/libbeat/processors/actions/lowercase.go new file mode 100644 index 00000000000..7439c7a0826 --- /dev/null +++ b/libbeat/processors/actions/lowercase.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "strings" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/checks" + conf "github.com/elastic/elastic-agent-libs/config" +) + +func init() { + processors.RegisterPlugin( + "lowercase", + checks.ConfigChecked( + NewLowerCaseProcessor, + checks.RequireFields("fields"), + checks.AllowedFields("fields", "when", "ignore_missing", "fail_on_error", "alter_full_field"), + ), + ) +} + +// NewLowerCaseProcessor converts event keys matching the provided fields to lowercase +func NewLowerCaseProcessor(c *conf.C) (beat.Processor, error) { + return NewAlterFieldProcessor(c, "lowercase", lowerCase) +} + +func lowerCase(field string) (string, error) { + return strings.ToLower(field), nil +} diff --git a/libbeat/processors/actions/lowercase_test.go b/libbeat/processors/actions/lowercase_test.go new file mode 100644 index 00000000000..cce5a03c37b --- /dev/null +++ b/libbeat/processors/actions/lowercase_test.go @@ -0,0 +1,327 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestNewLowerCaseProcessor(t *testing.T) { + c := conf.MustNewConfigFrom( + mapstr.M{ + "fields": []string{"field1", "type", "field2", "type.value.key", "typeKey"}, // "type" is our mandatory field + "ignore_missing": true, + "fail_on_error": false, + }, + ) + + procInt, err := NewLowerCaseProcessor(c) + assert.NoError(t, err) + + processor, ok := procInt.(*alterFieldProcessor) + assert.True(t, ok) + assert.Equal(t, []string{"field1", "field2", "typeKey"}, processor.Fields) // we discard both "type" and "type.value.key" as mandatory fields + assert.True(t, processor.IgnoreMissing) + assert.False(t, processor.FailOnError) +} + +func TestLowerCaseProcessorRun(t *testing.T) { + tests := []struct { + Name string + Fields []string + IgnoreMissing bool + FailOnError bool + FullPath bool + Input mapstr.M + Output mapstr.M + Error bool + }{ + { + Name: "Lowercase Fields", + Fields: []string{"a.b.c", "Field1"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "C": "D", + }, + }, + }, + Output: mapstr.M{ + "field1": mapstr.M{"Field2": "Value"}, // field1 is lowercased + "Field3": "Value", + "a": mapstr.M{ + "b": mapstr.M{ + "c": "D", + }, + }, + }, + Error: false, + }, + { + Name: "Lowercase Fields", + Fields: []string{"a.b.c", "Field1"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "C": "D", + }, + }, + }, + Output: mapstr.M{ + "field1": mapstr.M{"Field2": "Value"}, // field1 is lowercased + "Field3": "Value", + "a": mapstr.M{ + "b": mapstr.M{ + "c": "D", + }, + }, + }, + Error: false, + }, + { + Name: "Lowercase Fields when full_path is false", // searches only the most nested key 'case insensitively' + Fields: []string{"a.B.c"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: false, + Input: mapstr.M{ + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "C": "D", + }, + }, + }, + Output: mapstr.M{ + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "c": "D", // only c is lowercased + }, + }, + }, + + Error: false, + }, + { + Name: "Revert to original map on error", + Fields: []string{"Field1", "abcbd"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": "value1", + "ab": "first", + }, + Output: mapstr.M{ + "Field1": "value1", + "ab": "first", + "error": mapstr.M{"message": "could not fetch value for key: abcbd, Error: key not found"}, + }, + Error: true, + }, + { + Name: "Ignore Missing Key Error", + Fields: []string{"Field4"}, + IgnoreMissing: true, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Output: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Error: false, + }, + { + Name: "Do Not Fail On Missing Key Error", + Fields: []string{"Field4"}, + IgnoreMissing: false, + FailOnError: false, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Output: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Error: false, + }, + { + Name: "Fail On Missing Key Error", + Fields: []string{"Field4"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Output: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + "error": mapstr.M{"message": "could not fetch value for key: Field4, Error: key not found"}, + }, + Error: true, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + p := &alterFieldProcessor{ + Fields: test.Fields, + IgnoreMissing: test.IgnoreMissing, + FailOnError: test.FailOnError, + AlterFullField: test.FullPath, + alterFunc: lowerCase, + } + + event, err := p.Run(&beat.Event{Fields: test.Input}) + + if !test.Error { + require.NoError(t, err) + } else { + require.Error(t, err) + } + + assert.Equal(t, test.Output, event.Fields) + }) + } + + t.Run("test key collison", func(t *testing.T) { + Input := + mapstr.M{ + "ab": "first", + "Ab": "second", + } + + p := &alterFieldProcessor{ + Fields: []string{"ab"}, + IgnoreMissing: false, + FailOnError: true, + AlterFullField: true, + alterFunc: lowerCase, + } + + _, err := p.Run(&beat.Event{Fields: Input}) + require.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyCollision) + + }) +} + +func BenchmarkLowerCaseProcessorRun(b *testing.B) { + tests := []struct { + Name string + Events []beat.Event + }{ + { + Name: "5000 events with 5 fields on each level with 3 level depth without collisions", + Events: GenerateEvents(5000, 5, 3, false), + }, + { + Name: "5000 events with 5 fields on each level with 3 level depth with collisions", + Events: GenerateEvents(5000, 5, 3, true), + }, + { + Name: "500 events with 50 fields on each level with 5 level depth without collisions", + Events: GenerateEvents(500, 50, 3, false), + }, + { + Name: "500 events with 50 fields on each level with 5 level depth with collisions", + Events: GenerateEvents(500, 50, 3, true), + }, + // Add more test cases as needed for benchmarking + } + + for _, tt := range tests { + b.Run(tt.Name, func(b *testing.B) { + p := &alterFieldProcessor{ + Fields: []string{"level1field1.level2field1.level3field1"}, + alterFunc: lowerCase, + AlterFullField: true, + IgnoreMissing: false, + FailOnError: true, + } + for i := 0; i < b.N; i++ { + //Run the function with the input + for _, e := range tt.Events { + ev := e + _, err := p.Run(&ev) + require.NoError(b, err) + } + + } + }) + } +} + +func GenerateEvents(numEvents, fieldsPerLevel, depth int, withCollisions bool) []beat.Event { + events := make([]beat.Event, numEvents) + for i := 0; i < numEvents; i++ { + event := &beat.Event{Fields: mapstr.M{}} + generateFields(event, fieldsPerLevel, depth, withCollisions) + events[i] = *event + } + return events +} + +func generateFields(event *beat.Event, fieldsPerLevel, depth int, withCollisions bool) { + if depth == 0 { + return + } + + for j := 1; j <= fieldsPerLevel; j++ { + var key string + for d := 1; d < depth; d++ { + key += fmt.Sprintf("level%dfield%d", d, j) + key += "." + } + if withCollisions { + key += fmt.Sprintf("Level%dField%d", depth, j) // Creating a collision (Level is capitalized) + } else { + key += fmt.Sprintf("level%dfield%d", depth, j) + } + event.Fields.Put(key, "value") + key = "" + } + +}