From 8fccc4ad581cc0a6e8866615ec0682252cb0e6a1 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Tue, 25 Oct 2022 10:31:10 -0700 Subject: [PATCH] Allow to configure sampling config for logs Fixes https://github.com/open-telemetry/opentelemetry-collector/issues/4554 Signed-off-by: Bogdan --- .chloggen/allowsampling.yaml | 11 ++++++++++ service/config_provider_test.go | 10 +++++++--- .../configunmarshaler/defaultunmarshaler.go | 10 +++++++--- .../defaultunmarshaler_test.go | 20 +++++++++++++------ service/telemetry/config.go | 11 ++++++++++ service/telemetry/telemetry.go | 20 ++++++++++++------- 6 files changed, 63 insertions(+), 19 deletions(-) create mode 100755 .chloggen/allowsampling.yaml diff --git a/.chloggen/allowsampling.yaml b/.chloggen/allowsampling.yaml new file mode 100755 index 000000000000..7bca9951e458 --- /dev/null +++ b/.chloggen/allowsampling.yaml @@ -0,0 +1,11 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: service/telemetry + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Allow to configure sampling config for logs. + +# One or more tracking issues or pull requests related to the change +issues: [4554] diff --git a/service/config_provider_test.go b/service/config_provider_test.go index a6b7189975ba..230b54bea465 100644 --- a/service/config_provider_test.go +++ b/service/config_provider_test.go @@ -74,9 +74,13 @@ var configNop = &Config{ }, Telemetry: telemetry.Config{ Logs: telemetry.LogsConfig{ - Level: zapcore.InfoLevel, - Development: false, - Encoding: "console", + Level: zapcore.InfoLevel, + Development: false, + Encoding: "console", + Sampling: &telemetry.LogsSamplingConfig{ + Initial: 100, + Thereafter: 100, + }, OutputPaths: []string{"stderr"}, ErrorOutputPaths: []string{"stderr"}, DisableCaller: false, diff --git a/service/internal/configunmarshaler/defaultunmarshaler.go b/service/internal/configunmarshaler/defaultunmarshaler.go index 4ec6ae1fc549..1f8946b210b2 100644 --- a/service/internal/configunmarshaler/defaultunmarshaler.go +++ b/service/internal/configunmarshaler/defaultunmarshaler.go @@ -67,9 +67,13 @@ func Unmarshal(v *confmap.Conf, factories component.Factories) (*config.Config, Service: config.Service{ Telemetry: telemetry.Config{ Logs: telemetry.LogsConfig{ - Level: zapcore.InfoLevel, - Development: false, - Encoding: "console", + Level: zapcore.InfoLevel, + Development: false, + Encoding: "console", + Sampling: &telemetry.LogsSamplingConfig{ + Initial: 100, + Thereafter: 100, + }, OutputPaths: []string{"stderr"}, ErrorOutputPaths: []string{"stderr"}, DisableCaller: false, diff --git a/service/internal/configunmarshaler/defaultunmarshaler_test.go b/service/internal/configunmarshaler/defaultunmarshaler_test.go index b46c8205694b..915e8adf9669 100644 --- a/service/internal/configunmarshaler/defaultunmarshaler_test.go +++ b/service/internal/configunmarshaler/defaultunmarshaler_test.go @@ -61,9 +61,13 @@ func TestLoadEmptyAllSections(t *testing.T) { Encoding: "console", DisableCaller: zapProdCfg.DisableCaller, DisableStacktrace: zapProdCfg.DisableStacktrace, - OutputPaths: zapProdCfg.OutputPaths, - ErrorOutputPaths: zapProdCfg.ErrorOutputPaths, - InitialFields: zapProdCfg.InitialFields, + Sampling: &telemetry.LogsSamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + OutputPaths: zapProdCfg.OutputPaths, + ErrorOutputPaths: zapProdCfg.ErrorOutputPaths, + InitialFields: zapProdCfg.InitialFields, }, cfg.Service.Telemetry.Logs) } @@ -105,9 +109,13 @@ func TestUnmarshal(t *testing.T) { Encoding: "console", DisableCaller: true, DisableStacktrace: true, - OutputPaths: []string{"stderr", "./output-logs"}, - ErrorOutputPaths: []string{"stderr", "./error-output-logs"}, - InitialFields: map[string]interface{}{"field_key": "filed_value"}, + Sampling: &telemetry.LogsSamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + OutputPaths: []string{"stderr", "./output-logs"}, + ErrorOutputPaths: []string{"stderr", "./error-output-logs"}, + InitialFields: map[string]interface{}{"field_key": "filed_value"}, }, Metrics: telemetry.MetricsConfig{ Level: configtelemetry.LevelNormal, diff --git a/service/telemetry/config.go b/service/telemetry/config.go index 198dbef218f8..cc36dab69ccc 100644 --- a/service/telemetry/config.go +++ b/service/telemetry/config.go @@ -61,6 +61,9 @@ type LogsConfig struct { // (default = false) DisableStacktrace bool `mapstructure:"disable_stacktrace"` + // Sampling sets a sampling policy. A nil SamplingConfig disables sampling. + Sampling *LogsSamplingConfig `mapstructure:"sampling"` + // OutputPaths is a list of URLs or file paths to write logging output to. // The URLs could only be with "file" schema or without schema. // The URLs with "file" schema must be an absolute path. @@ -91,6 +94,14 @@ type LogsConfig struct { InitialFields map[string]interface{} `mapstructure:"initial_fields"` } +// LogsSamplingConfig sets a sampling strategy for the logger. Sampling caps the +// global CPU and I/O load that logging puts on your process while attempting +// to preserve a representative subset of your logs. +type LogsSamplingConfig struct { + Initial int `mapstructure:"initial"` + Thereafter int `mapstructure:"thereafter"` +} + // MetricsConfig exposes the common Telemetry configuration for one component. // Experimental: *NOTE* this structure is subject to change or removal in the future. type MetricsConfig struct { diff --git a/service/telemetry/telemetry.go b/service/telemetry/telemetry.go index 7aba69863098..25f3549eacf2 100644 --- a/service/telemetry/telemetry.go +++ b/service/telemetry/telemetry.go @@ -73,12 +73,9 @@ func New(_ context.Context, set Settings, cfg Config) (*Telemetry, error) { func newLogger(cfg LogsConfig, options []zap.Option) (*zap.Logger, error) { // Copied from NewProductionConfig. zapCfg := &zap.Config{ - Level: zap.NewAtomicLevelAt(cfg.Level), - Development: cfg.Development, - Sampling: &zap.SamplingConfig{ - Initial: 100, - Thereafter: 100, - }, + Level: zap.NewAtomicLevelAt(cfg.Level), + Development: cfg.Development, + Sampling: toSamplingConfig(cfg.Sampling), Encoding: cfg.Encoding, EncoderConfig: zap.NewProductionEncoderConfig(), OutputPaths: cfg.OutputPaths, @@ -101,9 +98,18 @@ func newLogger(cfg LogsConfig, options []zap.Option) (*zap.Logger, error) { return logger, nil } -type nopSpanProcessor struct { +func toSamplingConfig(sc *LogsSamplingConfig) *zap.SamplingConfig { + if sc == nil { + return nil + } + return &zap.SamplingConfig{ + Initial: sc.Initial, + Thereafter: sc.Thereafter, + } } +type nopSpanProcessor struct{} + func (n nopSpanProcessor) OnStart(context.Context, sdktrace.ReadWriteSpan) {} func (n nopSpanProcessor) OnEnd(sdktrace.ReadOnlySpan) {}