From 44ceeb8bef3c8776596c95fb193465353d18815b Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Tue, 6 Jun 2023 23:15:53 -0400 Subject: [PATCH] [pkg/ottl] Changed replacement string to be a path expression to a string telemetry field or a literal string --- .chloggen/ottl_function_changes.yaml | 20 +++++ pkg/ottl/ottlfuncs/README.md | 4 +- pkg/ottl/ottlfuncs/func_replace_match.go | 14 ++-- pkg/ottl/ottlfuncs/func_replace_match_test.go | 40 ++++++--- pkg/ottl/ottlfuncs/func_replace_pattern.go | 14 ++-- .../ottlfuncs/func_replace_pattern_test.go | 84 +++++++++++++------ 6 files changed, 128 insertions(+), 48 deletions(-) create mode 100755 .chloggen/ottl_function_changes.yaml diff --git a/.chloggen/ottl_function_changes.yaml b/.chloggen/ottl_function_changes.yaml new file mode 100755 index 000000000000..4d12b4a6430c --- /dev/null +++ b/.chloggen/ottl_function_changes.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Changed replacement_pattern (replacement) string to be a path expression to a string telemetry field or a literal string + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [22787] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 485d175bad2d..0a4105fcba4b 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -186,7 +186,7 @@ If using OTTL outside of collector configuration, `$` should not be escaped and The `replace_match` function allows replacing entire strings if they match a glob pattern. -`target` is a path expression to a telemetry field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is a string. +`target` is a path expression to a telemetry field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. If `target` matches `pattern` it will get replaced with `replacement`. @@ -200,7 +200,7 @@ Examples: The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. -`target` is a path expression to a telemetry field. `regex` is a regex string indicating a segment to replace. `replacement` is a string. +`target` is a path expression to a telemetry field. `regex` is a regex string indicating a segment to replace. `replacement` is either a path expression to a string telemetry field or a literal string. If one or more sections of `target` match `regex` they will get replaced with `replacement`. diff --git a/pkg/ottl/ottlfuncs/func_replace_match.go b/pkg/ottl/ottlfuncs/func_replace_match.go index ebab2bd3283f..7219692a5099 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match.go +++ b/pkg/ottl/ottlfuncs/func_replace_match.go @@ -13,9 +13,9 @@ import ( ) type ReplaceMatchArguments[K any] struct { - Target ottl.GetSetter[K] `ottlarg:"0"` - Pattern string `ottlarg:"1"` - Replacement string `ottlarg:"2"` + Target ottl.GetSetter[K] `ottlarg:"0"` + Pattern string `ottlarg:"1"` + Replacement ottl.StringGetter[K] `ottlarg:"2"` } func NewReplaceMatchFactory[K any]() ottl.Factory[K] { @@ -32,22 +32,26 @@ func createReplaceMatchFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argume return replaceMatch(args.Target, args.Pattern, args.Replacement) } -func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement string) (ottl.ExprFunc[K], error) { +func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) } return func(ctx context.Context, tCtx K) (interface{}, error) { val, err := target.Get(ctx, tCtx) + replacementVal, errRepl := replacement.Get(ctx, tCtx) if err != nil { return nil, err } + if errRepl != nil { + return nil, errRepl + } if val == nil { return nil, nil } if valStr, ok := val.(string); ok { if glob.Match(valStr) { - err = target.Set(ctx, tCtx, replacement) + err = target.Set(ctx, tCtx, replacementVal) if err != nil { return nil, err } diff --git a/pkg/ottl/ottlfuncs/func_replace_match_test.go b/pkg/ottl/ottlfuncs/func_replace_match_test.go index 2108d87ae75b..b2b726bf51fd 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_match_test.go @@ -30,23 +30,31 @@ func Test_replaceMatch(t *testing.T) { name string target ottl.GetSetter[pcommon.Value] pattern string - replacement string + replacement ottl.StringGetter[pcommon.Value] want func(pcommon.Value) }{ { - name: "replace match", - target: target, - pattern: "hello*", - replacement: "hello {universe}", + name: "replace match", + target: target, + pattern: "hello*", + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "hello {universe}", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello {universe}") }, }, { - name: "no match", - target: target, - pattern: "goodbye*", - replacement: "goodbye {universe}", + name: "no match", + target: target, + pattern: "goodbye*", + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "goodbye {universe}", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello world") }, @@ -81,8 +89,13 @@ func Test_replaceMatch_bad_input(t *testing.T) { return nil }, } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{replacement}", nil + }, + } - exprFunc, err := replaceMatch[interface{}](target, "*", "{replacement}") + exprFunc, err := replaceMatch[interface{}](target, "*", replacement) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -102,8 +115,13 @@ func Test_replaceMatch_get_nil(t *testing.T) { return nil }, } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{anything}", nil + }, + } - exprFunc, err := replaceMatch[interface{}](target, "*", "{anything}") + exprFunc, err := replaceMatch[interface{}](target, "*", replacement) assert.NoError(t, err) result, err := exprFunc(nil, nil) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 9ff3b76070f8..1a5cccd8c5fc 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -12,9 +12,9 @@ import ( ) type ReplacePatternArguments[K any] struct { - Target ottl.GetSetter[K] `ottlarg:"0"` - RegexPattern string `ottlarg:"1"` - Replacement string `ottlarg:"2"` + Target ottl.GetSetter[K] `ottlarg:"0"` + RegexPattern string `ottlarg:"1"` + Replacement ottl.StringGetter[K] `ottlarg:"2"` } func NewReplacePatternFactory[K any]() ottl.Factory[K] { @@ -31,22 +31,26 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return replacePattern(args.Target, args.RegexPattern, args.Replacement) } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement string) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) } return func(ctx context.Context, tCtx K) (interface{}, error) { originalVal, err := target.Get(ctx, tCtx) + replacementVal, errRepl := replacement.Get(ctx, tCtx) if err != nil { return nil, err } + if errRepl != nil { + return nil, errRepl + } if originalVal == nil { return nil, nil } if originalValStr, ok := originalVal.(string); ok { if compiledPattern.MatchString(originalValStr) { - updatedStr := compiledPattern.ReplaceAllString(originalValStr, replacement) + updatedStr := compiledPattern.ReplaceAllString(originalValStr, replacementVal) err = target.Set(ctx, tCtx, updatedStr) if err != nil { return nil, err diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 19427e8d0ecf..278cc2ccbb57 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -31,50 +31,70 @@ func Test_replacePattern(t *testing.T) { name string target ottl.GetSetter[pcommon.Value] pattern string - replacement string + replacement ottl.StringGetter[pcommon.Value] want func(pcommon.Value) }{ { - name: "replace regex match", - target: target, - pattern: `passwd\=[^\s]*(\s?)`, - replacement: "passwd=*** ", + name: "replace regex match", + target: target, + pattern: `passwd\=[^\s]*(\s?)`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=*** ", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") }, }, { - name: "no regex match", - target: target, - pattern: `nomatch\=[^\s]*(\s?)`, - replacement: "shouldnotbeinoutput", + name: "no regex match", + target: target, + pattern: `nomatch\=[^\s]*(\s?)`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "shouldnotbeinoutput", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") }, }, { - name: "multiple regex match", - target: target, - pattern: `key[^\s]*(\s?)`, - replacement: "**** ", + name: "multiple regex match", + target: target, + pattern: `key[^\s]*(\s?)`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "**** ", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") }, }, { - name: "expand capturing groups", - target: target, - pattern: `(\w+)=(\w+)`, - replacement: "$1:$2", + name: "expand capturing groups", + target: target, + pattern: `(\w+)=(\w+)`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "$1:$2", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") }, }, { - name: "replacement with literal $", - target: target, - pattern: `passwd\=[^\s]*(\s?)`, - replacement: "passwd=$$$$$$ ", + name: "replacement with literal $", + target: target, + pattern: `passwd\=[^\s]*(\s?)`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=$$$$$$ ", nil + }, + }, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") }, @@ -83,7 +103,6 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement) assert.NoError(t, err) @@ -110,8 +129,13 @@ func Test_replacePattern_bad_input(t *testing.T) { return nil }, } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{replacement}", nil + }, + } - exprFunc, err := replacePattern[interface{}](target, "regexp", "{replacement}") + exprFunc, err := replacePattern[interface{}](target, "regexp", replacement) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -130,8 +154,13 @@ func Test_replacePattern_get_nil(t *testing.T) { return nil }, } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{anything}", nil + }, + } - exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, "{anything}") + exprFunc, err := replacePattern[interface{}](target, `nomatch\=[^\s]*(\s?)`, replacement) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -150,9 +179,14 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { return nil }, } + replacement := &ottl.StandardStringGetter[interface{}]{ + Getter: func(context.Context, interface{}) (interface{}, error) { + return "{anything}", nil + }, + } invalidRegexPattern := "*" - _, err := replacePattern[interface{}](target, invalidRegexPattern, "{anything}") + _, err := replacePattern[interface{}](target, invalidRegexPattern, replacement) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") }