Skip to content
This repository has been archived by the owner on May 25, 2022. It is now read-only.

Commit

Permalink
Provide a dedicated mechanism for parsing logger name
Browse files Browse the repository at this point in the history
  • Loading branch information
djaglowski committed Mar 24, 2022
1 parent 6c3c01e commit bb44cea
Show file tree
Hide file tree
Showing 19 changed files with 764 additions and 34 deletions.
64 changes: 64 additions & 0 deletions docs/types/scope_name.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
## Scope Name Parsing

A Scope Name may be parsed from a log entry in order to indicate the code from which a log was emitted.

### `scope_name` parsing parameters

Parser operators can parse a scope name and attach the resulting value to a log entry.

| Field | Default | Description |
| --- | --- | --- |
| `parse_from` | required | The [field](/docs/types/field.md) from which the value will be parsed. |
| `preserve_to` | | Preserves the unparsed value at the specified [field](/docs/types/field.md). |


### How to use `scope_name` parsing

All parser operators, such as [`regex_parser`](/docs/operators/regex_parser.md) support these fields inside of a `scope_name` block.

If a `scope_name` block is specified, the parser operator will perform the parsing _after_ performing its other parsing actions, but _before_ passing the entry to the specified output operator.


### Example Configurations

#### Parse a scope_name from a string

Configuration:
```yaml
- type: regex_parser
regexp: '^(?P<scope_name_field>\S*)\s-\s(?P<message>.*)'
scope_name:
parse_from: body.scope_name_field
```
<table>
<tr><td> Input entry </td> <td> Output entry </td></tr>
<tr>
<td>
```json
{
"resource": { },
"attributes": { },
"body": "com.example.Foo - some message",
"scope_name": "",
}
```

</td>
<td>

```json
{
"resource": { },
"attributes": { },
"body": {
"message": "some message",
},
"scope_name": "com.example.Foo",
}
```

</td>
</tr>
</table>
2 changes: 2 additions & 0 deletions entry/entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type Entry struct {
TraceId []byte `json:"trace_id,omitempty" yaml:"trace_id,omitempty"`
TraceFlags []byte `json:"trace_flags,omitempty" yaml:"trace_flags,omitempty"`
Severity Severity `json:"severity" yaml:"severity"`
ScopeName string `json:"scope_name" yaml:"scope_name"`
}

// New will create a new log entry with current timestamp and an empty body.
Expand Down Expand Up @@ -185,5 +186,6 @@ func (entry *Entry) Copy() *Entry {
TraceId: copyByteArray(entry.TraceId),
SpanId: copyByteArray(entry.SpanId),
TraceFlags: copyByteArray(entry.TraceFlags),
ScopeName: entry.ScopeName,
}
}
5 changes: 5 additions & 0 deletions entry/entry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ func TestCopy(t *testing.T) {
entry.TraceId = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
entry.SpanId = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}
entry.TraceFlags = []byte{0x01}
entry.ScopeName = "my.logger"
copy := entry.Copy()

entry.Severity = Severity(1)
Expand All @@ -157,6 +158,7 @@ func TestCopy(t *testing.T) {
entry.TraceId[0] = 0xff
entry.SpanId[0] = 0xff
entry.TraceFlags[0] = 0xff
entry.ScopeName = "foo"

require.Equal(t, now, copy.ObservedTimestamp)
require.Equal(t, time.Time{}, copy.Timestamp)
Expand All @@ -168,6 +170,7 @@ func TestCopy(t *testing.T) {
require.Equal(t, []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, copy.TraceId)
require.Equal(t, []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, copy.SpanId)
require.Equal(t, []byte{0x01}, copy.TraceFlags)
require.Equal(t, "my.logger", copy.ScopeName)
}

func TestCopyNil(t *testing.T) {
Expand All @@ -185,6 +188,7 @@ func TestCopyNil(t *testing.T) {
entry.TraceId = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
entry.SpanId = []byte{0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x01, 0x02, 0x03}
entry.TraceFlags = []byte{0x01}
entry.ScopeName = "foo"

require.Equal(t, now, copy.ObservedTimestamp)
require.Equal(t, time.Time{}, copy.Timestamp)
Expand All @@ -196,6 +200,7 @@ func TestCopyNil(t *testing.T) {
require.Equal(t, []byte{}, copy.TraceId)
require.Equal(t, []byte{}, copy.SpanId)
require.Equal(t, []byte{}, copy.TraceFlags)
require.Equal(t, "", copy.ScopeName)
}

func TestFieldFromString(t *testing.T) {
Expand Down
22 changes: 16 additions & 6 deletions operator/helper/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type ParserConfig struct {
TimeParser *TimeParser `mapstructure:"timestamp,omitempty" json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
SeverityParserConfig *SeverityParserConfig `mapstructure:"severity,omitempty" json:"severity,omitempty" yaml:"severity,omitempty"`
TraceParser *TraceParser `mapstructure:"trace,omitempty" json:"trace,omitempty" yaml:"trace,omitempty"`
ScopeNameParser *ScopeNameParser `mapstructure:"scope_name,omitempty" json:"scope_name,omitempty" yaml:"scope_name,omitempty"`
}

// Build will build a parser operator.
Expand Down Expand Up @@ -87,12 +88,13 @@ func (c ParserConfig) Build(logger *zap.SugaredLogger) (ParserOperator, error) {
// ParserOperator provides a basic implementation of a parser operator.
type ParserOperator struct {
TransformerOperator
ParseFrom entry.Field
ParseTo entry.Field
PreserveTo *entry.Field
TimeParser *TimeParser
SeverityParser *SeverityParser
TraceParser *TraceParser
ParseFrom entry.Field
ParseTo entry.Field
PreserveTo *entry.Field
TimeParser *TimeParser
SeverityParser *SeverityParser
TraceParser *TraceParser
ScopeNameParser *ScopeNameParser
}

// ProcessWith will run ParseWith on the entry, then forward the entry on to the next operators.
Expand Down Expand Up @@ -169,6 +171,11 @@ func (p *ParserOperator) ParseWith(ctx context.Context, entry *entry.Entry, pars
traceParseErr = p.TraceParser.Parse(entry)
}

var logernameParserErr error
if p.ScopeNameParser != nil {
logernameParserErr = p.ScopeNameParser.Parse(entry)
}

// Handle time or severity parsing errors after attempting to parse both
if timeParseErr != nil {
return p.HandleEntryError(ctx, entry, errors.Wrap(timeParseErr, "time parser"))
Expand All @@ -179,6 +186,9 @@ func (p *ParserOperator) ParseWith(ctx context.Context, entry *entry.Entry, pars
if traceParseErr != nil {
return p.HandleEntryError(ctx, entry, errors.Wrap(traceParseErr, "trace parser"))
}
if logernameParserErr != nil {
return p.HandleEntryError(ctx, entry, errors.Wrap(logernameParserErr, "scope_name parser"))
}
return nil
}

Expand Down
33 changes: 22 additions & 11 deletions operator/helper/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,22 +421,27 @@ func TestParserPreserve(t *testing.T) {
}
}
func NewTestParserConfig() ParserConfig {
except := NewParserConfig("parser_config", "test_type")
except.ParseFrom = entry.NewBodyField("from")
except.ParseTo = entry.NewBodyField("to")
expect := NewParserConfig("parser_config", "test_type")
expect.ParseFrom = entry.NewBodyField("from")
expect.ParseTo = entry.NewBodyField("to")
tp := NewTimeParser()
expect.TimeParser = &tp

sp := NewSeverityParserConfig()
sp.Mapping = map[interface{}]interface{}{
"info": "3xx",
"warn": "4xx"}
except.TimeParser = &tp
except.SeverityParserConfig = &sp
"warn": "4xx",
}
expect.SeverityParserConfig = &sp

return except
lnp := NewScopeNameParser()
lnp.ParseFrom = entry.NewBodyField("logger")
expect.ScopeNameParser = &lnp
return expect
}

func TestMapStructureDecodeParserConfigWithHook(t *testing.T) {
except := NewTestParserConfig()
expect := NewTestParserConfig()
input := map[string]interface{}{
"id": "parser_config",
"type": "test_type",
Expand All @@ -452,6 +457,9 @@ func TestMapStructureDecodeParserConfigWithHook(t *testing.T) {
"warn": "4xx",
},
},
"scope_name": map[string]interface{}{
"parse_from": "body.logger",
},
}

var actual ParserConfig
Expand All @@ -460,11 +468,11 @@ func TestMapStructureDecodeParserConfigWithHook(t *testing.T) {
require.NoError(t, err)
err = ms.Decode(input)
require.NoError(t, err)
require.Equal(t, except, actual)
require.Equal(t, expect, actual)
}

func TestMapStructureDecodeParserConfig(t *testing.T) {
except := NewTestParserConfig()
expect := NewTestParserConfig()
input := map[string]interface{}{
"id": "parser_config",
"type": "test_type",
Expand All @@ -480,12 +488,15 @@ func TestMapStructureDecodeParserConfig(t *testing.T) {
"warn": "4xx",
},
},
"scope_name": map[string]interface{}{
"parse_from": entry.NewBodyField("logger"),
},
}

var actual ParserConfig
err := mapstructure.Decode(input, &actual)
require.NoError(t, err)
require.Equal(t, except, actual)
require.Equal(t, expect, actual)
}

func writerWithFakeOut(t *testing.T) (*WriterOperator, *testutil.FakeOutput) {
Expand Down
65 changes: 65 additions & 0 deletions operator/helper/scope_name.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helper

import (
"github.com/open-telemetry/opentelemetry-log-collection/entry"
"github.com/open-telemetry/opentelemetry-log-collection/errors"
)

// ScopeNameParser is a helper that parses severity onto an entry.
type ScopeNameParser struct {
ParseFrom entry.Field `mapstructure:"parse_from,omitempty" json:"parse_from,omitempty" yaml:"parse_from,omitempty"`
PreserveTo *entry.Field `mapstructure:"preserve_to,omitempty" json:"preserve_to,omitempty" yaml:"preserve_to,omitempty"`
}

// NewScopeNameParser creates a new scope parser with default values
func NewScopeNameParser() ScopeNameParser {
return ScopeNameParser{}
}

// Parse will parse severity from a field and attach it to the entry
func (p *ScopeNameParser) Parse(ent *entry.Entry) error {
value, ok := ent.Delete(p.ParseFrom)
if !ok {
return errors.NewError(
"log entry does not have the expected parse_from field",
"ensure that all entries forwarded to this parser contain the parse_from field",
"parse_from", p.ParseFrom.String(),
)
}

strVal, ok := value.(string)
if !ok {
err := ent.Set(p.ParseFrom, value)
if err != nil {
return errors.Wrap(err, "parse_from field does not contain a string")
}
return errors.NewError(
"parse_from field does not contain a string",
"ensure that all entries forwarded to this parser contain a string in the parse_from field",
"parse_from", p.ParseFrom.String(),
)
}

ent.ScopeName = strVal
if p.PreserveTo != nil {
if err := ent.Set(p.PreserveTo, value); err != nil {
return errors.Wrap(err, "set preserve_to")
}
}

return nil
}
Loading

0 comments on commit bb44cea

Please sign in to comment.