[8.x](backport #41424) [libbeat]: Add lowercase processor

CHANGELOG.next.asciidoc

@@ -245,7 +245,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add process.entity_id, process.group.name and process.group.id in add_process_metadata processor. Make fim module with kprobes backend to always add an appropriately configured add_process_metadata processor to enrich file events {pull}38776[38776]
 
 *Auditbeat*
-
+- Add `lowercase` processor. {issue}22254[22254] {pull}41424[41424]
 
 *Auditbeat*
 

NOTICE.txt

@@ -13443,11 +13443,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.12.1
+Version: v0.17.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.17.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -191,7 +191,7 @@ require (
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
 	github.com/elastic/elastic-agent-autodiscover v0.9.0
-	github.com/elastic/elastic-agent-libs v0.12.1
+	github.com/elastic/elastic-agent-libs v0.17.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.1
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/elastic/go-quark v0.2.0

go.sum

@@ -342,8 +342,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE=
 github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0=
-github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-libs v0.17.1 h1:1MXoc1eHGE8hCdVJ9+qiGiZAGeHzT2QBVVzD/oxwqeU=
+github.com/elastic/elastic-agent-libs v0.17.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.11.1 h1:BxViQHnqxvvi/65rj3mGwG6Eto6ldFCTnuDTUJnakaU=
 github.com/elastic/elastic-agent-system-metrics v0.11.1/go.mod h1:3QiMu9wTKJFvpCN+5klgGqasTMNKJbgY3xcoN1KQXJk=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=

libbeat/processors/actions/alterFieldProcessor.go

@@ -0,0 +1,135 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package actions
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/processors"
+	conf "github.com/elastic/elastic-agent-libs/config"
+	"github.com/elastic/elastic-agent-libs/mapstr"
+)
+
+type alterFieldProcessor struct {
+	Fields         []string
+	IgnoreMissing  bool
+	FailOnError    bool
+	AlterFullField bool
+
+	processorName string
+	alterFunc     mapstr.AlterFunc
+}
+
+// NewAlterFieldProcessor is an umbrella method for processing events based on provided fields. Such as converting event keys to uppercase/lowercase
+func NewAlterFieldProcessor(c *conf.C, processorName string, alterFunc mapstr.AlterFunc) (beat.Processor, error) {
+	config := struct {
+		Fields         []string `config:"fields"`
+		IgnoreMissing  bool     `config:"ignore_missing"`
+		FailOnError    bool     `config:"fail_on_error"`
+		AlterFullField bool     `config:"alter_full_field"`
+	}{
+		IgnoreMissing:  false,
+		FailOnError:    true,
+		AlterFullField: true,
+	}
+
+	if err := c.Unpack(&config); err != nil {
+		return nil, fmt.Errorf("failed to unpack the %s fields configuration: %w", processorName, err)
+	}
+
+	// Skip mandatory fields
+	var configFields []string
+	var lowerField string
+	for _, readOnly := range processors.MandatoryExportedFields {
+		readOnly = strings.ToLower(readOnly)
+		for _, field := range config.Fields {
+			// Skip fields that match "readOnly" or start with "readOnly."
+			lowerField = strings.ToLower(field)
+			if strings.HasPrefix(lowerField, readOnly+".") || lowerField == readOnly {
+				continue
+			}
+			// Add fields that do not match "readOnly" criteria
+			configFields = append(configFields, field)
+		}
+	}
+	return &alterFieldProcessor{
+		Fields:         configFields,
+		IgnoreMissing:  config.IgnoreMissing,
+		FailOnError:    config.FailOnError,
+		processorName:  processorName,
+		AlterFullField: config.AlterFullField,
+		alterFunc:      alterFunc,
+	}, nil
+
+}
+
+func (a *alterFieldProcessor) String() string {
+	return fmt.Sprintf("%s fields=%+v", a.processorName, *a)
+}
+
+func (a *alterFieldProcessor) Run(event *beat.Event) (*beat.Event, error) {
+	var backup *beat.Event
+	if a.FailOnError {
+		backup = event.Clone()
+	}
+
+	for _, field := range a.Fields {
+		err := a.alter(event, field)
+		if err != nil {
+			if a.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) {
+				continue
+			}
+			if a.FailOnError {
+				event = backup
+				_, _ = event.PutValue("error.message", err.Error())
+				return event, err
+			}
+		}
+	}
+
+	return event, nil
+}
+
+func (a *alterFieldProcessor) alter(event *beat.Event, field string) error {
+
+	// modify all segments of the key
+	if a.AlterFullField {
+		err := event.Fields.AlterPath(field, mapstr.CaseInsensitiveMode, a.alterFunc)
+		if err != nil {
+			return err
+		}
+	} else {
+		// modify only the last segment
+		segmentCount := strings.Count(field, ".")
+		err := event.Fields.AlterPath(field, mapstr.CaseInsensitiveMode, func(key string) (string, error) {
+			if segmentCount > 0 {
+				segmentCount--
+				return key, nil
+			}
+			return a.alterFunc(key)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

libbeat/processors/actions/docs/lowercase.asciidoc

@@ -0,0 +1,114 @@
+[[lowercase]]
+=== Lowercase fields in events
+
+++++
+<titleabbrev>lowercase</titleabbrev>
+++++
+
+The `lowercase` processor specifies a list of fields that should be converted to lowercase. This transformation applies to keys that match the specified fields. Matching is performed case-insensitively. 
+
+
+==== Examples: 
+
+1. Default scenario 
+
+[source,yaml]
+----
+processors:
+  - rename:
+      fields:
+        - "ab.cd"
+      ignore_missing: false
+      fail_on_error: true
+      full_path: true
+----
+[source,json]
+----
+// Input
+{
+  "AB": {"CD":"data"},
+  "CD": {"ef":"data"} 
+}
+
+
+// output
+{
+  "ab": {"cd":"data"},  // `AB.CD` -> `ab.cd`
+  "CD": {"ef":"data"}  
+}
+----
+
+[start=2]
+2. When `full_path` is false
+
+[source,yaml]
+----
+processors:
+  - rename:
+      fields:
+        - "ab.cd"
+      ignore_missing: false
+      fail_on_error: true
+      alter_full_field: false
+----
+
+[source,json]
+----
+// Input
+{
+  "AB": {"CD":"data"},
+  "CD": {"ef":"data"} 
+}
+
+
+// output
+{
+  "AB": {"cd":"data"},  // `AB.CD` -> `AB.cd` (only `cd` is lowercased)
+  "CD": {"ef":"data"}  
+}
+----
+
+[start=2]
+2. In case of non unique path to the key
+
+[source,yaml]
+----
+processors:
+  - rename:
+      fields:
+        - "ab"
+      ignore_missing: false
+      fail_on_error: true
+      alter_full_field: true
+----
+
+[source,json]
+----
+// Input
+{
+  "ab": "first",
+  "aB": "second"
+}
+
+// Output
+{
+  "ab": "first",
+  "aB": "second",
+  "err": "... Error: key collision"
+}
+----
+
+==== Configuration:
+
+The `lowercase` processor has the following configuration settings:
+
+`fields`:: The field names to lowercase. The match is case-insensitive, e.g. `a.b.c.d` would match `A.b.C.d` or `A.B.C.D`.
+`ignore_missing`:: (Optional) Indicates whether to ignore events that lack the source field.
+                    The default is `false`, which will fail processing of an event if a field is missing.
+`fail_on_error`:: (Optional) If set to `true` and an error occurs, the changes are reverted and the original event is returned.
+                    If set to `false`, processing continues if an error occurs. Default is `true`.
+`alter_full_field`:: (Optional) If set to `true`, the entire key path is lowercased. If set to `false` only the final part of the key path is lowercased. Default is true    
+
+
+
+See <<conditions>> for a list of supported conditions.

libbeat/processors/actions/lowercase.go

@@ -0,0 +1,47 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package actions
+
+import (
+	"strings"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/processors"
+	"github.com/elastic/beats/v7/libbeat/processors/checks"
+	conf "github.com/elastic/elastic-agent-libs/config"
+)
+
+func init() {
+	processors.RegisterPlugin(
+		"lowercase",
+		checks.ConfigChecked(
+			NewLowerCaseProcessor,
+			checks.RequireFields("fields"),
+			checks.AllowedFields("fields", "when", "ignore_missing", "fail_on_error", "alter_full_field"),
+		),
+	)
+}
+
+// NewLowerCaseProcessor converts event keys matching the provided fields to lowercase
+func NewLowerCaseProcessor(c *conf.C) (beat.Processor, error) {
+	return NewAlterFieldProcessor(c, "lowercase", lowerCase)
+}
+
+func lowerCase(field string) (string, error) {
+	return strings.ToLower(field), nil
+}