diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9264f45233b..684860cf64b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -240,7 +240,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942] - Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled - Add `lowercase` processor. {issue}22254[22254] {pull}41424[41424] - +- Add `uppercase` processor. {issue}22254[22254] {pull}41535[41535] *Auditbeat* - Added `add_session_metadata` processor, which enables session viewer on Auditbeat data. {pull}37640[37640] diff --git a/libbeat/processors/actions/docs/uppercase.asciidoc b/libbeat/processors/actions/docs/uppercase.asciidoc new file mode 100644 index 00000000000..b9ede3b35e9 --- /dev/null +++ b/libbeat/processors/actions/docs/uppercase.asciidoc @@ -0,0 +1,119 @@ +[[uppercase]] +=== Uppercase fields in events + +++++ +uppercase +++++ + +The `uppercase` processor specifies a list of `fields` and `values` to be converted to uppercase. Keys listed in `fields` will be matched case-insensitively and converted to uppercase. For `values`, only exact, case-sensitive matches are transformed to uppercase. This way, keys and values can be selectively converted based on the specified criteria. + + +==== Examples: + +1. Default scenario + +[source,yaml] +---- +processors: + - rename: + fields: + - "ab.cd" + values: + - "testKey" + ignore_missing: false + fail_on_error: true + alter_full_field: true +---- +[source,json] +---- +// Input +{ + "ab": {"cd":"data"}, + "CD": {"ef":"data"}, + "testKey": {"testvalue"} +} + + +// output +{ + "ab": {"cd":"data"}, // `ab.cd` -> `AB.CD` + "CD": {"ef":"data"}, + "testKey": {"TESTVALUE"} // `testvalue` -> `TESTVALUE` is uppercased +} +---- + +[start=2] +2. When `alter_full_field` is false (applicable only for fields) + +[source,yaml] +---- +processors: + - rename: + fields: + - "ab.cd" + ignore_missing: false + fail_on_error: true + alter_full_field: false +---- + +[source,json] +---- +// Input +{ + "ab": {"cd":"data"}, + "CD": {"ef":"data"}, +} + + +// output +{ + "ab": {"CD":"data"}, // `ab.cd` -> `ab.CD` (only `cd` is uppercased) + "CD": {"ef":"data"}, +} +---- + +[start=2] +2. In case of non unique path to the key + +[source,yaml] +---- +processors: + - rename: + fields: + - "ab" + ignore_missing: false + fail_on_error: true + alter_full_field: true +---- + +[source,json] +---- +// Input +{ + "ab": "first", + "aB": "second" +} + +// Output +{ + "ab": "first", + "aB": "second", + "err": "... Error: key collision" +} +---- + +==== Configuration: + +The `uppercase` processor has the following configuration settings: + +`fields`:: The field names to uppercase. The match is case-insensitive, e.g. `a.b.c.d` would match `A.b.C.d` or `A.B.C.D`. +`values`:: (Optional) Specifies the exact values to be converted to uppercase. Each entry should include the full path to the value. Key matching is case-sensitive. If the target value is not a string, an error is triggered (`fail_on_error: true`) or the value is skipped (`fail_on_error: false`). +`ignore_missing`:: (Optional) Indicates whether to ignore events that lack the source field. + The default is `false`, which will fail processing of an event if a field is missing. +`fail_on_error`:: (Optional) If set to `true` and an error occurs, the changes are reverted and the original event is returned. + If set to `false`, processing continues if an error occurs. Default is `true`. +`alter_full_field`:: (Optional) If set to `true`, the entire key path is uppercased. If set to `false` only the final part of the key path is uppercased. Default is true + + + +See <> for a list of supported conditions. diff --git a/libbeat/processors/actions/lowercase_test.go b/libbeat/processors/actions/lowercase_test.go index 6dba685caa4..855112094fe 100644 --- a/libbeat/processors/actions/lowercase_test.go +++ b/libbeat/processors/actions/lowercase_test.go @@ -59,32 +59,6 @@ func TestLowerCaseProcessorRun(t *testing.T) { Output mapstr.M Error bool }{ - { - Name: "Lowercase Fields", - Fields: []string{"a.b.c", "Field1"}, - IgnoreMissing: false, - FailOnError: true, - FullPath: true, - Input: mapstr.M{ - "Field1": mapstr.M{"Field2": "Value"}, - "Field3": "Value", - "a": mapstr.M{ - "B": mapstr.M{ - "C": "D", - }, - }, - }, - Output: mapstr.M{ - "field1": mapstr.M{"Field2": "Value"}, // field1 is lowercased - "Field3": "Value", - "a": mapstr.M{ - "b": mapstr.M{ - "c": "D", - }, - }, - }, - Error: false, - }, { Name: "Lowercase Fields", Fields: []string{"a.b.c", "Field1"}, diff --git a/libbeat/processors/actions/uppercase.go b/libbeat/processors/actions/uppercase.go new file mode 100644 index 00000000000..ad00df603d8 --- /dev/null +++ b/libbeat/processors/actions/uppercase.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "strings" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/libbeat/processors/checks" + conf "github.com/elastic/elastic-agent-libs/config" +) + +func init() { + processors.RegisterPlugin( + "uppercase", + checks.ConfigChecked( + NewUpperCaseProcessor, + checks.RequireFields("fields"), + checks.AllowedFields("fields", "ignore_missing", "fail_on_error", "alter_full_field", "values"), + ), + ) +} + +// NewUpperCaseProcessor converts event keys matching the provided fields to uppercase +func NewUpperCaseProcessor(c *conf.C) (beat.Processor, error) { + return NewAlterFieldProcessor(c, "uppercase", upperCase) +} + +func upperCase(field string) (string, error) { + return strings.ToUpper(field), nil +} diff --git a/libbeat/processors/actions/uppercase_test.go b/libbeat/processors/actions/uppercase_test.go new file mode 100644 index 00000000000..2e643eadaaf --- /dev/null +++ b/libbeat/processors/actions/uppercase_test.go @@ -0,0 +1,193 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestNewUpperCaseProcessor(t *testing.T) { + c := conf.MustNewConfigFrom( + mapstr.M{ + "fields": []string{"field1", "type", "field2", "type.value.key", "typeKey"}, // "type" is our mandatory field + "ignore_missing": true, + "fail_on_error": false, + }, + ) + + procInt, err := NewUpperCaseProcessor(c) + assert.NoError(t, err) + + processor, ok := procInt.(*alterFieldProcessor) + assert.True(t, ok) + assert.Equal(t, []string{"field1", "field2", "typeKey"}, processor.Fields) // we discard both "type" and "type.value.key" as mandatory fields + assert.True(t, processor.IgnoreMissing) + assert.False(t, processor.FailOnError) +} + +func TestUpperCaseProcessorRun(t *testing.T) { + tests := []struct { + Name string + Fields []string + Values []string + IgnoreMissing bool + FailOnError bool + FullPath bool + Input mapstr.M + Output mapstr.M + Error bool + }{ + { + Name: "Uppercase Fields", + Fields: []string{"a.b.c", "Field1"}, + Values: []string{"Field3"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "C": "D", + }, + }, + }, + Output: mapstr.M{ + "FIELD1": mapstr.M{"Field2": "Value"}, // FIELD1 is uppercased + "Field3": "VALUE", // VALUE is uppercased + "A": mapstr.M{ + "B": mapstr.M{ + "C": "D", + }, + }, + }, + Error: false, + }, + { + Name: "Uppercase Fields when full_path is false", // searches only the most nested key 'case insensitively' + Fields: []string{"a.B.c"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: false, + Input: mapstr.M{ + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "c": "D", + }, + }, + }, + Output: mapstr.M{ + "Field3": "Value", + "a": mapstr.M{ + "B": mapstr.M{ + "C": "D", // only c is uppercased + }, + }, + }, + + Error: false, + }, + { + Name: "Fail On Missing Key Error", + Fields: []string{"Field4"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Output: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + "error": mapstr.M{"message": "could not fetch value for key: Field4, Error: key not found"}, + }, + Error: true, + }, + { + Name: "Fail if value is not a string", + Values: []string{"Field1"}, + IgnoreMissing: false, + FailOnError: true, + FullPath: true, + Input: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + }, + Output: mapstr.M{ + "Field1": mapstr.M{"Field2": "Value"}, + "Field3": "Value", + "error": mapstr.M{"message": "value of key \"Field1\" is not a string"}, + }, + Error: true, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + p := &alterFieldProcessor{ + Fields: test.Fields, + Values: test.Values, + IgnoreMissing: test.IgnoreMissing, + FailOnError: test.FailOnError, + AlterFullField: test.FullPath, + alterFunc: upperCase, + } + + event, err := p.Run(&beat.Event{Fields: test.Input}) + + if !test.Error { + require.NoError(t, err) + } else { + require.Error(t, err) + } + + assert.Equal(t, test.Output, event.Fields) + }) + } + + t.Run("test key collison", func(t *testing.T) { + Input := + mapstr.M{ + "ab": "first", + "Ab": "second", + } + + p := &alterFieldProcessor{ + Fields: []string{"ab"}, + IgnoreMissing: false, + FailOnError: true, + AlterFullField: true, + alterFunc: upperCase, + } + + _, err := p.Run(&beat.Event{Fields: Input}) + require.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyCollision) + + }) +}