From 90e31d9b64b3ee6bf251984973fbee24d608fa6a Mon Sep 17 00:00:00 2001 From: Daniel Jaglowski Date: Thu, 24 Mar 2022 13:34:33 -0400 Subject: [PATCH] Provide a dedicated mechanism for parsing logger name (#397) --- docs/types/scope_name.md | 64 +++++ entry/entry.go | 2 + entry/entry_test.go | 5 + operator/helper/parser.go | 22 +- operator/helper/parser_test.go | 33 ++- operator/helper/scope_name.go | 65 +++++ operator/helper/scope_name_test.go | 242 ++++++++++++++++++ .../testdata/scope_name/parse_from.yaml | 1 + .../testdata/scope_name/preserve_to.yaml | 2 + operator/output/stdout/stdout_test.go | 2 +- operator/parser/json/config_test.go | 18 +- operator/parser/json/json_test.go | 74 ++++++ operator/parser/json/testdata/scope_name.yaml | 3 + operator/parser/regex/config_test.go | 20 +- .../parser/regex/testdata/scope_name.yaml | 4 + operator/parser/scope/scope_name.go | 67 +++++ operator/parser/scope/scope_name_test.go | 163 ++++++++++++ operator/parser/severity/severity.go | 6 +- operator/parser/time/time.go | 5 - 19 files changed, 764 insertions(+), 34 deletions(-) create mode 100644 docs/types/scope_name.md create mode 100644 operator/helper/scope_name.go create mode 100644 operator/helper/scope_name_test.go create mode 100644 operator/helper/testdata/scope_name/parse_from.yaml create mode 100644 operator/helper/testdata/scope_name/preserve_to.yaml create mode 100644 operator/parser/json/testdata/scope_name.yaml create mode 100644 operator/parser/regex/testdata/scope_name.yaml create mode 100644 operator/parser/scope/scope_name.go create mode 100644 operator/parser/scope/scope_name_test.go diff --git a/docs/types/scope_name.md b/docs/types/scope_name.md new file mode 100644 index 00000000..7a71fac0 --- /dev/null +++ b/docs/types/scope_name.md @@ -0,0 +1,64 @@ +## Scope Name Parsing + +A Scope Name may be parsed from a log entry in order to indicate the code from which a log was emitted. + +### `scope_name` parsing parameters + +Parser operators can parse a scope name and attach the resulting value to a log entry. + +| Field | Default | Description | +| --- | --- | --- | +| `parse_from` | required | The [field](/docs/types/field.md) from which the value will be parsed. | +| `preserve_to` | | Preserves the unparsed value at the specified [field](/docs/types/field.md). | + + +### How to use `scope_name` parsing + +All parser operators, such as [`regex_parser`](/docs/operators/regex_parser.md) support these fields inside of a `scope_name` block. + +If a `scope_name` block is specified, the parser operator will perform the parsing _after_ performing its other parsing actions, but _before_ passing the entry to the specified output operator. + + +### Example Configurations + +#### Parse a scope_name from a string + +Configuration: +```yaml +- type: regex_parser + regexp: '^(?P\S*)\s-\s(?P.*)' + scope_name: + parse_from: body.scope_name_field +``` + + + + + + + +
Input entry Output entry
+ +```json +{ + "resource": { }, + "attributes": { }, + "body": "com.example.Foo - some message", + "scope_name": "", +} +``` + + + +```json +{ + "resource": { }, + "attributes": { }, + "body": { + "message": "some message", + }, + "scope_name": "com.example.Foo", +} +``` + +
diff --git a/entry/entry.go b/entry/entry.go index 931d549c..37dd6e3e 100644 --- a/entry/entry.go +++ b/entry/entry.go @@ -33,6 +33,7 @@ type Entry struct { TraceId []byte `json:"trace_id,omitempty" yaml:"trace_id,omitempty"` TraceFlags []byte `json:"trace_flags,omitempty" yaml:"trace_flags,omitempty"` Severity Severity `json:"severity" yaml:"severity"` + ScopeName string `json:"scope_name" yaml:"scope_name"` } // New will create a new log entry with current timestamp and an empty body. @@ -185,5 +186,6 @@ func (entry *Entry) Copy() *Entry { TraceId: copyByteArray(entry.TraceId), SpanId: copyByteArray(entry.SpanId), TraceFlags: copyByteArray(entry.TraceFlags), + ScopeName: entry.ScopeName, } } diff --git a/entry/entry_test.go b/entry/entry_test.go index d735d813..0c8eac9e 100644 --- a/entry/entry_test.go +++ b/entry/entry_test.go @@ -146,6 +146,7 @@ func TestCopy(t *testing.T) { entry.TraceId = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} entry.SpanId = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08} entry.TraceFlags = []byte{0x01} + entry.ScopeName = "my.logger" copy := entry.Copy() entry.Severity = Severity(1) @@ -157,6 +158,7 @@ func TestCopy(t *testing.T) { entry.TraceId[0] = 0xff entry.SpanId[0] = 0xff entry.TraceFlags[0] = 0xff + entry.ScopeName = "foo" require.Equal(t, now, copy.ObservedTimestamp) require.Equal(t, time.Time{}, copy.Timestamp) @@ -168,6 +170,7 @@ func TestCopy(t *testing.T) { require.Equal(t, []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, copy.TraceId) require.Equal(t, []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, copy.SpanId) require.Equal(t, []byte{0x01}, copy.TraceFlags) + require.Equal(t, "my.logger", copy.ScopeName) } func TestCopyNil(t *testing.T) { @@ -185,6 +188,7 @@ func TestCopyNil(t *testing.T) { entry.TraceId = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} entry.SpanId = []byte{0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x01, 0x02, 0x03} entry.TraceFlags = []byte{0x01} + entry.ScopeName = "foo" require.Equal(t, now, copy.ObservedTimestamp) require.Equal(t, time.Time{}, copy.Timestamp) @@ -196,6 +200,7 @@ func TestCopyNil(t *testing.T) { require.Equal(t, []byte{}, copy.TraceId) require.Equal(t, []byte{}, copy.SpanId) require.Equal(t, []byte{}, copy.TraceFlags) + require.Equal(t, "", copy.ScopeName) } func TestFieldFromString(t *testing.T) { diff --git a/operator/helper/parser.go b/operator/helper/parser.go index ba5f9e9d..98dbbc01 100644 --- a/operator/helper/parser.go +++ b/operator/helper/parser.go @@ -43,6 +43,7 @@ type ParserConfig struct { TimeParser *TimeParser `mapstructure:"timestamp,omitempty" json:"timestamp,omitempty" yaml:"timestamp,omitempty"` SeverityParserConfig *SeverityParserConfig `mapstructure:"severity,omitempty" json:"severity,omitempty" yaml:"severity,omitempty"` TraceParser *TraceParser `mapstructure:"trace,omitempty" json:"trace,omitempty" yaml:"trace,omitempty"` + ScopeNameParser *ScopeNameParser `mapstructure:"scope_name,omitempty" json:"scope_name,omitempty" yaml:"scope_name,omitempty"` } // Build will build a parser operator. @@ -87,12 +88,13 @@ func (c ParserConfig) Build(logger *zap.SugaredLogger) (ParserOperator, error) { // ParserOperator provides a basic implementation of a parser operator. type ParserOperator struct { TransformerOperator - ParseFrom entry.Field - ParseTo entry.Field - PreserveTo *entry.Field - TimeParser *TimeParser - SeverityParser *SeverityParser - TraceParser *TraceParser + ParseFrom entry.Field + ParseTo entry.Field + PreserveTo *entry.Field + TimeParser *TimeParser + SeverityParser *SeverityParser + TraceParser *TraceParser + ScopeNameParser *ScopeNameParser } // ProcessWith will run ParseWith on the entry, then forward the entry on to the next operators. @@ -169,6 +171,11 @@ func (p *ParserOperator) ParseWith(ctx context.Context, entry *entry.Entry, pars traceParseErr = p.TraceParser.Parse(entry) } + var logernameParserErr error + if p.ScopeNameParser != nil { + logernameParserErr = p.ScopeNameParser.Parse(entry) + } + // Handle time or severity parsing errors after attempting to parse both if timeParseErr != nil { return p.HandleEntryError(ctx, entry, errors.Wrap(timeParseErr, "time parser")) @@ -179,6 +186,9 @@ func (p *ParserOperator) ParseWith(ctx context.Context, entry *entry.Entry, pars if traceParseErr != nil { return p.HandleEntryError(ctx, entry, errors.Wrap(traceParseErr, "trace parser")) } + if logernameParserErr != nil { + return p.HandleEntryError(ctx, entry, errors.Wrap(logernameParserErr, "scope_name parser")) + } return nil } diff --git a/operator/helper/parser_test.go b/operator/helper/parser_test.go index ab69265c..198f7a89 100644 --- a/operator/helper/parser_test.go +++ b/operator/helper/parser_test.go @@ -421,22 +421,27 @@ func TestParserPreserve(t *testing.T) { } } func NewTestParserConfig() ParserConfig { - except := NewParserConfig("parser_config", "test_type") - except.ParseFrom = entry.NewBodyField("from") - except.ParseTo = entry.NewBodyField("to") + expect := NewParserConfig("parser_config", "test_type") + expect.ParseFrom = entry.NewBodyField("from") + expect.ParseTo = entry.NewBodyField("to") tp := NewTimeParser() + expect.TimeParser = &tp + sp := NewSeverityParserConfig() sp.Mapping = map[interface{}]interface{}{ "info": "3xx", - "warn": "4xx"} - except.TimeParser = &tp - except.SeverityParserConfig = &sp + "warn": "4xx", + } + expect.SeverityParserConfig = &sp - return except + lnp := NewScopeNameParser() + lnp.ParseFrom = entry.NewBodyField("logger") + expect.ScopeNameParser = &lnp + return expect } func TestMapStructureDecodeParserConfigWithHook(t *testing.T) { - except := NewTestParserConfig() + expect := NewTestParserConfig() input := map[string]interface{}{ "id": "parser_config", "type": "test_type", @@ -452,6 +457,9 @@ func TestMapStructureDecodeParserConfigWithHook(t *testing.T) { "warn": "4xx", }, }, + "scope_name": map[string]interface{}{ + "parse_from": "body.logger", + }, } var actual ParserConfig @@ -460,11 +468,11 @@ func TestMapStructureDecodeParserConfigWithHook(t *testing.T) { require.NoError(t, err) err = ms.Decode(input) require.NoError(t, err) - require.Equal(t, except, actual) + require.Equal(t, expect, actual) } func TestMapStructureDecodeParserConfig(t *testing.T) { - except := NewTestParserConfig() + expect := NewTestParserConfig() input := map[string]interface{}{ "id": "parser_config", "type": "test_type", @@ -480,12 +488,15 @@ func TestMapStructureDecodeParserConfig(t *testing.T) { "warn": "4xx", }, }, + "scope_name": map[string]interface{}{ + "parse_from": entry.NewBodyField("logger"), + }, } var actual ParserConfig err := mapstructure.Decode(input, &actual) require.NoError(t, err) - require.Equal(t, except, actual) + require.Equal(t, expect, actual) } func writerWithFakeOut(t *testing.T) (*WriterOperator, *testutil.FakeOutput) { diff --git a/operator/helper/scope_name.go b/operator/helper/scope_name.go new file mode 100644 index 00000000..f766187c --- /dev/null +++ b/operator/helper/scope_name.go @@ -0,0 +1,65 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helper + +import ( + "github.com/open-telemetry/opentelemetry-log-collection/entry" + "github.com/open-telemetry/opentelemetry-log-collection/errors" +) + +// ScopeNameParser is a helper that parses severity onto an entry. +type ScopeNameParser struct { + ParseFrom entry.Field `mapstructure:"parse_from,omitempty" json:"parse_from,omitempty" yaml:"parse_from,omitempty"` + PreserveTo *entry.Field `mapstructure:"preserve_to,omitempty" json:"preserve_to,omitempty" yaml:"preserve_to,omitempty"` +} + +// NewScopeNameParser creates a new scope parser with default values +func NewScopeNameParser() ScopeNameParser { + return ScopeNameParser{} +} + +// Parse will parse severity from a field and attach it to the entry +func (p *ScopeNameParser) Parse(ent *entry.Entry) error { + value, ok := ent.Delete(p.ParseFrom) + if !ok { + return errors.NewError( + "log entry does not have the expected parse_from field", + "ensure that all entries forwarded to this parser contain the parse_from field", + "parse_from", p.ParseFrom.String(), + ) + } + + strVal, ok := value.(string) + if !ok { + err := ent.Set(p.ParseFrom, value) + if err != nil { + return errors.Wrap(err, "parse_from field does not contain a string") + } + return errors.NewError( + "parse_from field does not contain a string", + "ensure that all entries forwarded to this parser contain a string in the parse_from field", + "parse_from", p.ParseFrom.String(), + ) + } + + ent.ScopeName = strVal + if p.PreserveTo != nil { + if err := ent.Set(p.PreserveTo, value); err != nil { + return errors.Wrap(err, "set preserve_to") + } + } + + return nil +} diff --git a/operator/helper/scope_name_test.go b/operator/helper/scope_name_test.go new file mode 100644 index 00000000..e8e1ae0b --- /dev/null +++ b/operator/helper/scope_name_test.go @@ -0,0 +1,242 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helper + +import ( + "fmt" + "io/ioutil" + "path" + "testing" + "time" + + "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v2" + + "github.com/open-telemetry/opentelemetry-log-collection/entry" +) + +const testScopeName = "my.logger" + +func TestScopeNameParser(t *testing.T) { + now := time.Now() + testCases := []struct { + name string + parser *ScopeNameParser + input *entry.Entry + expectErr bool + expected *entry.Entry + }{ + { + name: "root_string", + parser: &ScopeNameParser{ + ParseFrom: entry.NewBodyField(), + }, + input: func() *entry.Entry { + e := entry.New() + e.Body = testScopeName + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "root_string_preserve", + parser: func() *ScopeNameParser { + preserve := entry.NewBodyField() + return &ScopeNameParser{ + ParseFrom: entry.NewBodyField(), + PreserveTo: &preserve, + } + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = testScopeName + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.Body = testScopeName + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "nondestructive_error", + parser: &ScopeNameParser{ + ParseFrom: entry.NewBodyField(), + }, + input: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + expectErr: true, + expected: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "nonroot_string", + parser: &ScopeNameParser{ + ParseFrom: entry.NewBodyField("logger"), + }, + input: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{} + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "nonroot_string_preserve", + parser: func() *ScopeNameParser { + preserve := entry.NewBodyField("somewhere") + return &ScopeNameParser{ + ParseFrom: entry.NewBodyField("logger"), + PreserveTo: &preserve, + } + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"somewhere": testScopeName} + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.parser.Parse(tc.input) + if tc.expectErr { + require.Error(t, err) + } + if tc.expected != nil { + require.Equal(t, tc.expected, tc.input) + } + }) + } +} + +func TestGoldenScopeNameParserConfig(t *testing.T) { + cases := []struct { + name string + expectErr bool + expect *ScopeNameParser + }{ + { + "parse_from", + false, + func() *ScopeNameParser { + cfg := NewScopeNameParser() + cfg.ParseFrom = entry.NewBodyField("from") + return &cfg + }(), + }, + { + "preserve_to", + false, + func() *ScopeNameParser { + cfg := NewScopeNameParser() + cfg.ParseFrom = entry.NewBodyField("from") + preserve := entry.NewBodyField("aField") + cfg.PreserveTo = &preserve + return &cfg + }(), + }, + } + + for _, tc := range cases { + t.Run("yaml/"+tc.name, func(t *testing.T) { + cfgFromYaml, yamlErr := ScopeConfigFromFileViaYaml(path.Join(".", "testdata", "scope_name", fmt.Sprintf("%s.yaml", tc.name))) + if tc.expectErr { + require.Error(t, yamlErr) + } else { + require.NoError(t, yamlErr) + require.Equal(t, tc.expect, cfgFromYaml) + } + }) + t.Run("mapstructure/"+tc.name, func(t *testing.T) { + parser := NewScopeNameParser() + cfgFromMapstructure := &parser + mapErr := ScopeConfigFromFileViaMapstructure( + path.Join(".", "testdata", "scope_name", fmt.Sprintf("%s.yaml", tc.name)), + cfgFromMapstructure, + ) + if tc.expectErr { + require.Error(t, mapErr) + } else { + require.NoError(t, mapErr) + require.Equal(t, tc.expect, cfgFromMapstructure) + } + }) + } +} + +func ScopeConfigFromFileViaYaml(file string) (*ScopeNameParser, error) { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("could not find config file: %s", err) + } + + parser := NewScopeNameParser() + config := &parser + if err := yaml.Unmarshal(bytes, config); err != nil { + return nil, fmt.Errorf("failed to read config file as yaml: %s", err) + } + + return config, nil +} + +func ScopeConfigFromFileViaMapstructure(file string, result *ScopeNameParser) error { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return fmt.Errorf("could not find config file: %s", err) + } + + raw := map[string]interface{}{} + + if err := yaml.Unmarshal(bytes, raw); err != nil { + return fmt.Errorf("failed to read data from yaml: %s", err) + } + + err = UnmarshalMapstructure(raw, result) + return err +} diff --git a/operator/helper/testdata/scope_name/parse_from.yaml b/operator/helper/testdata/scope_name/parse_from.yaml new file mode 100644 index 00000000..12247acb --- /dev/null +++ b/operator/helper/testdata/scope_name/parse_from.yaml @@ -0,0 +1 @@ +parse_from: body.from diff --git a/operator/helper/testdata/scope_name/preserve_to.yaml b/operator/helper/testdata/scope_name/preserve_to.yaml new file mode 100644 index 00000000..539b3591 --- /dev/null +++ b/operator/helper/testdata/scope_name/preserve_to.yaml @@ -0,0 +1,2 @@ +parse_from: body.from +preserve_to: body.aField diff --git a/operator/output/stdout/stdout_test.go b/operator/output/stdout/stdout_test.go index eb7f2f5a..bd383b38 100644 --- a/operator/output/stdout/stdout_test.go +++ b/operator/output/stdout/stdout_test.go @@ -60,6 +60,6 @@ func TestStdoutOperator(t *testing.T) { marshalledTs, err := json.Marshal(ts) require.NoError(t, err) - expected := `{"observed_timestamp":` + string(marshalledOTS) + `,"timestamp":` + string(marshalledTs) + `,"body":"test body","severity":0}` + "\n" + expected := `{"observed_timestamp":` + string(marshalledOTS) + `,"timestamp":` + string(marshalledTs) + `,"body":"test body","severity":0,"scope_name":""}` + "\n" require.Equal(t, expected, buf.String()) } diff --git a/operator/parser/json/config_test.go b/operator/parser/json/config_test.go index 60133756..21fa33df 100644 --- a/operator/parser/json/config_test.go +++ b/operator/parser/json/config_test.go @@ -70,16 +70,16 @@ func TestJSONParserConfig(t *testing.T) { Expect: func() *JSONParserConfig { cfg := defaultCfg() parseField := entry.NewBodyField("severity_field") - severityField := helper.NewSeverityParserConfig() - severityField.ParseFrom = &parseField + severityParser := helper.NewSeverityParserConfig() + severityParser.ParseFrom = &parseField mapping := map[interface{}]interface{}{ "critical": "5xx", "error": "4xx", "info": "3xx", "debug": "2xx", } - severityField.Mapping = mapping - cfg.SeverityParserConfig = &severityField + severityParser.Mapping = mapping + cfg.SeverityParserConfig = &severityParser return cfg }(), }, @@ -92,6 +92,16 @@ func TestJSONParserConfig(t *testing.T) { return cfg }(), }, + { + Name: "scope_name", + Expect: func() *JSONParserConfig { + cfg := defaultCfg() + loggerNameParser := helper.NewScopeNameParser() + loggerNameParser.ParseFrom = entry.NewBodyField("logger_name_field") + cfg.ScopeNameParser = &loggerNameParser + return cfg + }(), + }, } for _, tc := range cases { diff --git a/operator/parser/json/json_test.go b/operator/parser/json/json_test.go index 950e35d6..76516869 100644 --- a/operator/parser/json/json_test.go +++ b/operator/parser/json/json_test.go @@ -229,6 +229,80 @@ func TestJSONParserWithEmbeddedTimeParser(t *testing.T) { } } +func TestJSONParserWithEmbeddedScopeNameParser(t *testing.T) { + cases := []struct { + name string + inputBody map[string]interface{} + outputBody map[string]interface{} + errorExpected bool + preserveTo *entry.Field + }{ + { + "simple", + map[string]interface{}{ + "testfield": `{"logger_name":"logger"}`, + }, + map[string]interface{}{ + "testparsed": map[string]interface{}{}, + }, + false, + nil, + }, + { + "preserve", + map[string]interface{}{ + "testfield": `{"logger_name":"logger"}`, + }, + map[string]interface{}{ + "testparsed": map[string]interface{}{}, + "original_logger_name": "logger", + }, + false, + func() *entry.Field { + f := entry.NewBodyField("original_logger_name") + return &f + }(), + }, + { + "nested", + map[string]interface{}{ + "testfield": `{"superkey":"superval","logger_name":"logger"}`, + }, + map[string]interface{}{ + "testparsed": map[string]interface{}{ + "superkey": "superval", + }, + }, + false, + nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + input := entry.New() + input.Body = tc.inputBody + + output := entry.New() + output.Body = tc.outputBody + + parser, mockOutput := NewFakeJSONOperator() + parseFrom := entry.NewBodyField("testparsed", "logger_name") + parser.ParserOperator.ScopeNameParser = &helper.ScopeNameParser{ + ParseFrom: parseFrom, + PreserveTo: tc.preserveTo, + } + mockOutput.On("Process", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + e := args[1].(*entry.Entry) + require.Equal(t, tc.outputBody, e.Body) + }).Return(nil) + + err := parser.Process(context.Background(), input) + require.NoError(t, err) + }) + } +} + func TestJsonParserConfig(t *testing.T) { expect := NewJSONParserConfig("test") expect.ParseFrom = entry.NewBodyField("from") diff --git a/operator/parser/json/testdata/scope_name.yaml b/operator/parser/json/testdata/scope_name.yaml new file mode 100644 index 00000000..5b673ce8 --- /dev/null +++ b/operator/parser/json/testdata/scope_name.yaml @@ -0,0 +1,3 @@ +type: json_parser +scope_name: + parse_from: body.logger_name_field diff --git a/operator/parser/regex/config_test.go b/operator/parser/regex/config_test.go index b969ea77..09fe0c9b 100644 --- a/operator/parser/regex/config_test.go +++ b/operator/parser/regex/config_test.go @@ -70,16 +70,16 @@ func TestRegexParserGoldenConfig(t *testing.T) { Expect: func() *RegexParserConfig { cfg := defaultCfg() parseField := entry.NewBodyField("severity_field") - severityField := helper.NewSeverityParserConfig() - severityField.ParseFrom = &parseField + severityParser := helper.NewSeverityParserConfig() + severityParser.ParseFrom = &parseField mapping := map[interface{}]interface{}{ "critical": "5xx", "error": "4xx", "info": "3xx", "debug": "2xx", } - severityField.Mapping = mapping - cfg.SeverityParserConfig = &severityField + severityParser.Mapping = mapping + cfg.SeverityParserConfig = &severityParser return cfg }(), }, @@ -100,6 +100,18 @@ func TestRegexParserGoldenConfig(t *testing.T) { return cfg }(), }, + { + Name: "scope_name", + Expect: func() *RegexParserConfig { + cfg := defaultCfg() + cfg.Regex = "^Host=(?P[^,]+), Logger=(?P.*)$" + parseField := entry.NewBodyField("logger_name_field") + loggerNameParser := helper.NewScopeNameParser() + loggerNameParser.ParseFrom = parseField + cfg.ScopeNameParser = &loggerNameParser + return cfg + }(), + }, } for _, tc := range cases { diff --git a/operator/parser/regex/testdata/scope_name.yaml b/operator/parser/regex/testdata/scope_name.yaml new file mode 100644 index 00000000..7b89bafb --- /dev/null +++ b/operator/parser/regex/testdata/scope_name.yaml @@ -0,0 +1,4 @@ +type: regex_parser +regex: '^Host=(?P[^,]+), Logger=(?P.*)$' +scope_name: + parse_from: body.logger_name_field diff --git a/operator/parser/scope/scope_name.go b/operator/parser/scope/scope_name.go new file mode 100644 index 00000000..06ee0a82 --- /dev/null +++ b/operator/parser/scope/scope_name.go @@ -0,0 +1,67 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scope + +import ( + "context" + + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-log-collection/entry" + "github.com/open-telemetry/opentelemetry-log-collection/operator" + "github.com/open-telemetry/opentelemetry-log-collection/operator/helper" +) + +func init() { + operator.Register("scope_name_parser", func() operator.Builder { return NewScopeNameParserConfig("") }) +} + +// NewScopeNameParserConfig creates a new logger name parser config with default values +func NewScopeNameParserConfig(operatorID string) *ScopeNameParserConfig { + return &ScopeNameParserConfig{ + TransformerConfig: helper.NewTransformerConfig(operatorID, "scope_name_parser"), + ScopeNameParser: helper.NewScopeNameParser(), + } +} + +// ScopeNameParserConfig is the configuration of a logger name parser operator. +type ScopeNameParserConfig struct { + helper.TransformerConfig `mapstructure:",squash" yaml:",inline"` + helper.ScopeNameParser `mapstructure:",omitempty,squash" yaml:",omitempty,inline"` +} + +// Build will build a logger name parser operator. +func (c ScopeNameParserConfig) Build(logger *zap.SugaredLogger) (operator.Operator, error) { + transformerOperator, err := c.TransformerConfig.Build(logger) + if err != nil { + return nil, err + } + + return &ScopeNameParserOperator{ + TransformerOperator: transformerOperator, + ScopeNameParser: c.ScopeNameParser, + }, nil +} + +// ScopeNameParserOperator is an operator that parses logger name from a field to an entry. +type ScopeNameParserOperator struct { + helper.TransformerOperator + helper.ScopeNameParser +} + +// Process will parse logger name from an entry. +func (p *ScopeNameParserOperator) Process(ctx context.Context, entry *entry.Entry) error { + return p.ProcessWith(ctx, entry, p.Parse) +} diff --git a/operator/parser/scope/scope_name_test.go b/operator/parser/scope/scope_name_test.go new file mode 100644 index 00000000..b3ff17c6 --- /dev/null +++ b/operator/parser/scope/scope_name_test.go @@ -0,0 +1,163 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scope + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/open-telemetry/opentelemetry-log-collection/entry" + "github.com/open-telemetry/opentelemetry-log-collection/testutil" +) + +const testScopeName = "my.logger" + +func TestScopeNameParser(t *testing.T) { + now := time.Now() + testCases := []struct { + name string + config *ScopeNameParserConfig + input *entry.Entry + expectErr bool + expected *entry.Entry + }{ + { + name: "root_string", + config: func() *ScopeNameParserConfig { + cfg := NewScopeNameParserConfig("test") + cfg.ParseFrom = entry.NewBodyField() + return cfg + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = testScopeName + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "root_string_preserve", + config: func() *ScopeNameParserConfig { + cfg := NewScopeNameParserConfig("test") + cfg.ParseFrom = entry.NewBodyField() + preserve := entry.NewBodyField() + cfg.PreserveTo = &preserve + return cfg + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = testScopeName + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.Body = testScopeName + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "nondestructive_error", + config: func() *ScopeNameParserConfig { + cfg := NewScopeNameParserConfig("test") + cfg.ParseFrom = entry.NewBodyField() + return cfg + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + expectErr: true, + expected: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "nonroot_string", + config: func() *ScopeNameParserConfig { + cfg := NewScopeNameParserConfig("test") + cfg.ParseFrom = entry.NewBodyField("logger") + return cfg + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{} + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + { + name: "nonroot_string_preserve", + config: func() *ScopeNameParserConfig { + cfg := NewScopeNameParserConfig("test") + cfg.ParseFrom = entry.NewBodyField("logger") + preserve := entry.NewBodyField("somewhere") + cfg.PreserveTo = &preserve + return cfg + }(), + input: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"logger": testScopeName} + e.ObservedTimestamp = now + return e + }(), + expected: func() *entry.Entry { + e := entry.New() + e.Body = map[string]interface{}{"somewhere": testScopeName} + e.ScopeName = testScopeName + e.ObservedTimestamp = now + return e + }(), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + parser, err := tc.config.Build(testutil.Logger(t)) + require.NoError(t, err) + + err = parser.Process(context.Background(), tc.input) + if tc.expectErr { + require.Error(t, err) + } + if tc.expected != nil { + require.Equal(t, tc.expected, tc.input) + } + }) + } +} diff --git a/operator/parser/severity/severity.go b/operator/parser/severity/severity.go index 2f07b57f..625eeb1a 100644 --- a/operator/parser/severity/severity.go +++ b/operator/parser/severity/severity.go @@ -42,7 +42,7 @@ type SeverityParserConfig struct { helper.SeverityParserConfig `mapstructure:",omitempty,squash" yaml:",omitempty,inline"` } -// Build will build a time parser operator. +// Build will build a severity parser operator. func (c SeverityParserConfig) Build(logger *zap.SugaredLogger) (operator.Operator, error) { transformerOperator, err := c.TransformerConfig.Build(logger) if err != nil { @@ -60,13 +60,13 @@ func (c SeverityParserConfig) Build(logger *zap.SugaredLogger) (operator.Operato }, nil } -// SeverityParserOperator is an operator that parses time from a field to an entry. +// SeverityParserOperator is an operator that parses severity from a field to an entry. type SeverityParserOperator struct { helper.TransformerOperator helper.SeverityParser } -// Process will parse time from an entry. +// Process will parse severity from an entry. func (p *SeverityParserOperator) Process(ctx context.Context, entry *entry.Entry) error { return p.ProcessWith(ctx, entry, p.Parse) } diff --git a/operator/parser/time/time.go b/operator/parser/time/time.go index 57f2ae73..808ac94e 100644 --- a/operator/parser/time/time.go +++ b/operator/parser/time/time.go @@ -65,11 +65,6 @@ type TimeParserOperator struct { helper.TimeParser } -// CanOutput will always return true for a parser operator. -func (t *TimeParserOperator) CanOutput() bool { - return true -} - // Process will parse time from an entry. func (t *TimeParserOperator) Process(ctx context.Context, entry *entry.Entry) error { return t.ProcessWith(ctx, entry, t.TimeParser.Parse)