From b7c405df3b4900005e8420ada17cc8b518cbddbb Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Fri, 19 Nov 2021 14:51:51 -0500 Subject: [PATCH 01/23] Add ConfigSource component type --- component/configsource.go | 41 +++++++ component/factories.go | 2 + config/configmapprovider/default.go | 4 +- config/configmapprovider/default_test.go | 10 +- config/configmapprovider/simple.go | 17 +++ config/configsource.go | 58 ++++++++++ .../configunmarshaler/defaultunmarshaler.go | 25 +++-- configsource/files/config.go | 11 ++ configsource/files/factory.go | 34 ++++++ service/collector_windows.go | 4 +- service/command.go | 5 +- service/config_watcher.go | 24 +++- service/defaultconfigprovider.go | 104 ++++++++++++++++++ 13 files changed, 314 insertions(+), 25 deletions(-) create mode 100644 component/configsource.go create mode 100644 config/configsource.go create mode 100644 configsource/files/config.go create mode 100644 configsource/files/factory.go create mode 100644 service/defaultconfigprovider.go diff --git a/component/configsource.go b/component/configsource.go new file mode 100644 index 00000000000..c8a55972872 --- /dev/null +++ b/component/configsource.go @@ -0,0 +1,41 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package component // import "go.opentelemetry.io/collector/component" + +import ( + "context" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" +) + +// ConfigSource is +type ConfigSource interface { + configmapprovider.Provider +} + +// ConfigSourceCreateSettings is passed to ExtensionFactory.Create* functions. +type ConfigSourceCreateSettings struct { + //TelemetrySettings + + // BuildInfo can be used by components for informational purposes + BuildInfo BuildInfo +} + +type ConfigSourceFactory interface { + Factory + CreateDefaultConfig() config.ConfigSource + CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (ConfigSource, error) +} diff --git a/component/factories.go b/component/factories.go index 972e103538c..6682dff041c 100644 --- a/component/factories.go +++ b/component/factories.go @@ -34,6 +34,8 @@ type Factories struct { // Extensions maps extension type names in the config to the respective factory. Extensions map[config.Type]ExtensionFactory + + ConfigSources map[config.Type]ConfigSourceFactory } // MakeReceiverFactoryMap takes a list of receiver factories and returns a map diff --git a/config/configmapprovider/default.go b/config/configmapprovider/default.go index 35d9cfcd924..ad62986e4da 100644 --- a/config/configmapprovider/default.go +++ b/config/configmapprovider/default.go @@ -14,9 +14,9 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" -// NewDefault returns the default Provider, and it creates configuration from a file +// NewLocal returns the default Provider, and it creates configuration from a file // defined by the given configFile and overwrites fields using properties. -func NewDefault(configFileName string, properties []string) Provider { +func NewLocal(configFileName string, properties []string) Provider { return NewExpand( NewMerge( NewFile(configFileName), diff --git a/config/configmapprovider/default_test.go b/config/configmapprovider/default_test.go index 1f400f2330e..b6228d6655d 100644 --- a/config/configmapprovider/default_test.go +++ b/config/configmapprovider/default_test.go @@ -26,7 +26,7 @@ import ( ) func TestDefaultMapProvider(t *testing.T) { - mp := NewDefault("testdata/default-config.yaml", nil) + mp := NewLocal("testdata/default-config.yaml", nil) retr, err := mp.Retrieve(context.Background(), nil) require.NoError(t, err) @@ -45,7 +45,7 @@ exporters: } func TestDefaultMapProvider_AddNewConfig(t *testing.T) { - mp := NewDefault("testdata/default-config.yaml", []string{"processors.batch.timeout=2s"}) + mp := NewLocal("testdata/default-config.yaml", []string{"processors.batch.timeout=2s"}) cp, err := mp.Retrieve(context.Background(), nil) require.NoError(t, err) @@ -65,7 +65,7 @@ exporters: } func TestDefaultMapProvider_OverwriteConfig(t *testing.T) { - mp := NewDefault( + mp := NewLocal( "testdata/default-config.yaml", []string{"processors.batch.timeout=2s", "exporters.otlp.endpoint=localhost:1234"}) cp, err := mp.Retrieve(context.Background(), nil) @@ -87,7 +87,7 @@ exporters: } func TestDefaultMapProvider_InexistentFile(t *testing.T) { - mp := NewDefault("testdata/otelcol-config.yaml", nil) + mp := NewLocal("testdata/otelcol-config.yaml", nil) require.NotNil(t, mp) _, err := mp.Retrieve(context.Background(), nil) require.Error(t, err) @@ -96,7 +96,7 @@ func TestDefaultMapProvider_InexistentFile(t *testing.T) { } func TestDefaultMapProvider_EmptyFileName(t *testing.T) { - mp := NewDefault("", nil) + mp := NewLocal("", nil) _, err := mp.Retrieve(context.Background(), nil) require.Error(t, err) diff --git a/config/configmapprovider/simple.go b/config/configmapprovider/simple.go index e289e795002..3069933233e 100644 --- a/config/configmapprovider/simple.go +++ b/config/configmapprovider/simple.go @@ -20,6 +20,23 @@ import ( "go.opentelemetry.io/collector/config" ) +// TODO: This probably will make sense to be exported, but needs better name and documentation. +type simpleProvider struct { + confMap *config.Map +} + +func (s simpleProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { + return &simpleRetrieved{confMap: s.confMap}, nil +} + +func (s simpleProvider) Shutdown(ctx context.Context) error { + return nil +} + +func NewSimple(confMap *config.Map) Provider { + return &simpleProvider{confMap: confMap} +} + // TODO: This probably will make sense to be exported, but needs better name and documentation. type simpleRetrieved struct { confMap *config.Map diff --git a/config/configsource.go b/config/configsource.go new file mode 100644 index 00000000000..9cc8a666bc2 --- /dev/null +++ b/config/configsource.go @@ -0,0 +1,58 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config // import "go.opentelemetry.io/collector/config" + +// ConfigSource is the configuration of a component.ConfigSource. Specific ConfigSources must implement +// this interface and must embed ConfigSourceSettings struct or a struct that extends it. +type ConfigSource interface { + identifiable + validatable + + privateConfigConfigSource() +} + +// ConfigSourceSettings defines common settings for a component.ConfigSource configuration. +// Specific processors can embed this struct and extend it with more fields if needed. +// +// It is highly recommended to "override" the Validate() function. +// +// When embedded in the ConfigSource config, it must be with `mapstructure:",squash"` tag. +type ConfigSourceSettings struct { + id ComponentID `mapstructure:"-"` +} + +// NewConfigSourceSettings return a new ConfigSourceSettings with the given ComponentID. +func NewConfigSourceSettings(id ComponentID) ConfigSourceSettings { + return ConfigSourceSettings{id: ComponentID{typeVal: id.Type(), nameVal: id.Name()}} +} + +var _ ConfigSource = (*ConfigSourceSettings)(nil) + +// ID returns the receiver ComponentID. +func (es *ConfigSourceSettings) ID() ComponentID { + return es.id +} + +// SetIDName sets the receiver name. +func (es *ConfigSourceSettings) SetIDName(idName string) { + es.id.nameVal = idName +} + +// Validate validates the configuration and returns an error if invalid. +func (es *ConfigSourceSettings) Validate() error { + return nil +} + +func (es *ConfigSourceSettings) privateConfigConfigSource() {} diff --git a/config/configunmarshaler/defaultunmarshaler.go b/config/configunmarshaler/defaultunmarshaler.go index 9374e73615a..345e95ea104 100644 --- a/config/configunmarshaler/defaultunmarshaler.go +++ b/config/configunmarshaler/defaultunmarshaler.go @@ -66,11 +66,13 @@ const ( ) type configSettings struct { - Receivers map[config.ComponentID]map[string]interface{} `mapstructure:"receivers"` - Processors map[config.ComponentID]map[string]interface{} `mapstructure:"processors"` - Exporters map[config.ComponentID]map[string]interface{} `mapstructure:"exporters"` - Extensions map[config.ComponentID]map[string]interface{} `mapstructure:"extensions"` - Service map[string]interface{} `mapstructure:"service"` + Receivers map[config.ComponentID]map[string]interface{} `mapstructure:"receivers"` + Processors map[config.ComponentID]map[string]interface{} `mapstructure:"processors"` + Exporters map[config.ComponentID]map[string]interface{} `mapstructure:"exporters"` + Extensions map[config.ComponentID]map[string]interface{} `mapstructure:"extensions"` + ConfigSources map[config.ComponentID]map[string]interface{} `mapstructure:"config_sources"` + ValueSources map[config.ComponentID]map[string]interface{} `mapstructure:"value_sources"` + Service map[string]interface{} `mapstructure:"service"` } type defaultUnmarshaler struct{} @@ -84,6 +86,7 @@ func NewDefault() ConfigUnmarshaler { // Unmarshal the Config from a config.Map. // After the config is unmarshalled, `Validate()` must be called to validate. func (*defaultUnmarshaler) Unmarshal(v *config.Map, factories component.Factories) (*config.Config, error) { + var cfg config.Config // Unmarshal top level sections and validate. @@ -152,7 +155,7 @@ func unmarshalExtensions(exts map[config.ComponentID]map[string]interface{}, fac // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. - if err := unmarshal(config.NewMapFromStringMap(value), extensionCfg); err != nil { + if err := Unmarshal(config.NewMapFromStringMap(value), extensionCfg); err != nil { return nil, errorUnmarshalError(extensionsKeyName, id, err) } @@ -175,7 +178,7 @@ func unmarshalService(srvRaw map[string]interface{}) (config.Service, error) { }, } - if err := unmarshal(config.NewMapFromStringMap(srvRaw), &srv); err != nil { + if err := Unmarshal(config.NewMapFromStringMap(srvRaw), &srv); err != nil { return srv, fmt.Errorf("error reading service configuration: %w", err) } @@ -195,7 +198,7 @@ func LoadReceiver(componentConfig *config.Map, id config.ComponentID, factory co // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. - if err := unmarshal(componentConfig, receiverCfg); err != nil { + if err := Unmarshal(componentConfig, receiverCfg); err != nil { return nil, errorUnmarshalError(receiversKeyName, id, err) } @@ -244,7 +247,7 @@ func unmarshalExporters(exps map[config.ComponentID]map[string]interface{}, fact // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. - if err := unmarshal(config.NewMapFromStringMap(value), exporterCfg); err != nil { + if err := Unmarshal(config.NewMapFromStringMap(value), exporterCfg); err != nil { return nil, errorUnmarshalError(exportersKeyName, id, err) } @@ -272,7 +275,7 @@ func unmarshalProcessors(procs map[config.ComponentID]map[string]interface{}, fa // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. - if err := unmarshal(config.NewMapFromStringMap(value), processorCfg); err != nil { + if err := Unmarshal(config.NewMapFromStringMap(value), processorCfg); err != nil { return nil, errorUnmarshalError(processorsKeyName, id, err) } @@ -282,7 +285,7 @@ func unmarshalProcessors(procs map[config.ComponentID]map[string]interface{}, fa return processors, nil } -func unmarshal(componentSection *config.Map, intoCfg interface{}) error { +func Unmarshal(componentSection *config.Map, intoCfg interface{}) error { if cu, ok := intoCfg.(config.Unmarshallable); ok { return cu.Unmarshal(componentSection) } diff --git a/configsource/files/config.go b/configsource/files/config.go new file mode 100644 index 00000000000..2bfc73a0d80 --- /dev/null +++ b/configsource/files/config.go @@ -0,0 +1,11 @@ +package files + +import ( + "go.opentelemetry.io/collector/config" +) + +// Config defines configuration for logging exporter. +type Config struct { + config.ConfigSourceSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct + Name string +} diff --git a/configsource/files/factory.go b/configsource/files/factory.go new file mode 100644 index 00000000000..4e2a091c347 --- /dev/null +++ b/configsource/files/factory.go @@ -0,0 +1,34 @@ +package files + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" + "go.opentelemetry.io/collector/internal/internalinterface" +) + +type factory struct { + internalinterface.BaseInternal +} + +func (f factory) Type() config.Type { + return "files" +} + +func (f factory) CreateDefaultConfig() config.ConfigSource { + return &Config{ + ConfigSourceSettings: config.NewConfigSourceSettings(config.NewComponentID(f.Type())), + } +} + +func (f factory) CreateConfigSource( + ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource, +) (component.ConfigSource, error) { + return configmapprovider.NewFile(cfg.(*Config).Name), nil +} + +func NewFactory() component.ConfigSourceFactory { + return &factory{} +} diff --git a/service/collector_windows.go b/service/collector_windows.go index 87e5f3b4c87..10f8ceda55d 100644 --- a/service/collector_windows.go +++ b/service/collector_windows.go @@ -27,8 +27,6 @@ import ( "go.uber.org/zap/zapcore" "golang.org/x/sys/windows/svc" "golang.org/x/sys/windows/svc/eventlog" - - "go.opentelemetry.io/collector/config/configmapprovider" ) type WindowsService struct { @@ -133,7 +131,7 @@ func openEventLog(serviceName string) (*eventlog.Log, error) { func newWithWindowsEventLogCore(set CollectorSettings, elog *eventlog.Log) (*Collector, error) { if set.ConfigMapProvider == nil { - set.ConfigMapProvider = configmapprovider.NewDefault(getConfigFlag(), getSetFlag()) + set.ConfigMapProvider = NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) } set.LoggingOptions = append( set.LoggingOptions, diff --git a/service/command.go b/service/command.go index c449b93ed52..0d21a4fd885 100644 --- a/service/command.go +++ b/service/command.go @@ -16,8 +16,6 @@ package service // import "go.opentelemetry.io/collector/service" import ( "github.com/spf13/cobra" - - "go.opentelemetry.io/collector/config/configmapprovider" ) // NewCommand constructs a new cobra.Command using the given Collector. @@ -29,7 +27,8 @@ func NewCommand(set CollectorSettings) *cobra.Command { SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { if set.ConfigMapProvider == nil { - set.ConfigMapProvider = configmapprovider.NewDefault(getConfigFlag(), getSetFlag()) + //set.ConfigMapProvider = configmapprovider.NewLocal(getConfigFlag(), getSetFlag()) + set.ConfigMapProvider = NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) } col, err := New(set) if err != nil { diff --git a/service/config_watcher.go b/service/config_watcher.go index b8bff8a7698..43377fff23f 100644 --- a/service/config_watcher.go +++ b/service/config_watcher.go @@ -37,11 +37,24 @@ func newConfigWatcher(ctx context.Context, set CollectorSettings) (*configWatche return nil, fmt.Errorf("cannot retrieve the configuration: %w", err) } - var cfg *config.Config m, err := ret.Get(ctx) if err != nil { return nil, fmt.Errorf("cannot get the configuration: %w", err) } + + //sources, err := set.ConfigUnmarshaler.UnmarshalSources(m, set.Factories) + //if err != nil { + // return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) + //} + // + //cfgMap := config.NewMap() + //err := retrieveFromSources(ctx, cfgMap, sources, cm.onChange) + //if err != nil { + // return nil, fmt.Errorf("cannot retrieve the configuration: %w", err) + //} + //err := mergeConfigMap(cfgMap, m) + + var cfg *config.Config if cfg, err = set.ConfigUnmarshaler.Unmarshal(m, set.Factories); err != nil { return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) } @@ -56,6 +69,15 @@ func newConfigWatcher(ctx context.Context, set CollectorSettings) (*configWatche return cm, nil } +//func retrieveFromSources(ctx context.Context, cfgMap *config.Map, sources []component.ConfigSource, onChange func(*configmapprovider.ChangeEvent)) error { +// for _, source := range sources { +// retrieved, err := source.Retrieve(ctx, onChange) +// m, err := retrieved.Get(ctx) +// err := mergeConfigMap(cfgMap, m) +// } +// return nil +//} + func (cm *configWatcher) onChange(event *configmapprovider.ChangeEvent) { if event.Error != configsource.ErrSessionClosed { cm.watcher <- event.Error diff --git a/service/defaultconfigprovider.go b/service/defaultconfigprovider.go new file mode 100644 index 00000000000..7e747d65d6f --- /dev/null +++ b/service/defaultconfigprovider.go @@ -0,0 +1,104 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service // import "go.opentelemetry.io/collector/config/configmapprovider" + +import ( + "context" + "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configmapprovider" + "go.opentelemetry.io/collector/config/configunmarshaler" + + "go.opentelemetry.io/collector/config" +) + +type defaultConfigProvider struct { + provider configmapprovider.Provider + merged configmapprovider.Provider + factories component.Factories +} + +func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.Provider { + localProvider := configmapprovider.NewLocal(configFileName, properties) + return &defaultConfigProvider{provider: localProvider, factories: factories} +} + +func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(event *configmapprovider.ChangeEvent)) (configmapprovider.Retrieved, error) { + r, err := mp.provider.Retrieve(ctx, onChange) + if err != nil { + return nil, err + } + rootMap, err := r.Get(ctx) + + sources, err := unmarshalSources(ctx, rootMap, mp.factories) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) + } + + sources = append(sources, configmapprovider.NewSimple(rootMap)) + mp.merged = configmapprovider.NewMerge(sources...) + + return mp.merged.Retrieve(ctx, onChange) +} + +func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ([]configmapprovider.Provider, error) { + // Unmarshal the "config_sources" section of rootMap and create a Provider for each + // config source using the factories.ConfigSources. + + var providers []configmapprovider.Provider + + type RootConfig struct { + ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` + } + var rootCfg RootConfig + err := rootMap.Unmarshal(&rootCfg) + if err != nil { + return nil, err + } + + for _, sourceCfg := range rootCfg.ConfigSources { + for sourceType, settings := range sourceCfg { + factory, ok := factories.ConfigSources[config.Type(sourceType)] + if !ok { + return nil, fmt.Errorf("unknown source type %q", sourceType) + } + + cfg := factory.CreateDefaultConfig() + cfg.SetIDName(sourceType) + + // Now that the default config struct is created we can Unmarshal into it, + // and it will apply user-defined config on top of the default. + if err := configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { + return nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) // errorUnmarshalError(extensionsKeyName, id, err) + } + + source, err := factory.CreateConfigSource(ctx, component.ConfigSourceCreateSettings{}, cfg) + if err != nil { + return nil, err + } + providers = append(providers, source) + } + } + + return providers, nil +} + +func (mp *defaultConfigProvider) Shutdown(ctx context.Context) error { + if mp.merged != nil { + return mp.merged.Shutdown(ctx) + } + return nil +} From 9c893c08677a3c75423c50480d421ec38ac75aa4 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Fri, 19 Nov 2021 17:27:14 -0500 Subject: [PATCH 02/23] Add ValueSource --- component/configsource.go | 10 +- component/factories.go | 2 +- config/configmapprovider/configsource.go | 65 +++ config/configmapprovider/default.go | 4 +- config/configmapprovider/expand.go | 10 +- config/configmapprovider/file.go | 6 +- config/configmapprovider/inmemory.go | 6 +- config/configmapprovider/merge.go | 8 +- config/configmapprovider/properties.go | 6 +- config/configmapprovider/provider.go | 61 +-- config/configmapprovider/simple.go | 4 +- config/configmapprovider/valuesource.go | 79 +++ configsource/env/config.go | 10 + configsource/env/configsource.go | 32 ++ configsource/env/factory.go | 34 ++ configsource/files/factory.go | 2 +- service/command.go | 3 +- service/config_watcher.go | 2 +- service/config_watcher_test.go | 10 +- .../defaultconfigprovider.go | 39 +- .../valuesourcesubstitutor.go | 463 ++++++++++++++++++ service/settings.go | 2 +- 22 files changed, 752 insertions(+), 106 deletions(-) create mode 100644 config/configmapprovider/configsource.go create mode 100644 config/configmapprovider/valuesource.go create mode 100644 configsource/env/config.go create mode 100644 configsource/env/configsource.go create mode 100644 configsource/env/factory.go rename service/{ => defaultconfigprovider}/defaultconfigprovider.go (70%) create mode 100644 service/defaultconfigprovider/valuesourcesubstitutor.go diff --git a/component/configsource.go b/component/configsource.go index c8a55972872..e6bcce4cef4 100644 --- a/component/configsource.go +++ b/component/configsource.go @@ -23,7 +23,7 @@ import ( // ConfigSource is type ConfigSource interface { - configmapprovider.Provider + configmapprovider.Shutdownable } // ConfigSourceCreateSettings is passed to ExtensionFactory.Create* functions. @@ -34,8 +34,12 @@ type ConfigSourceCreateSettings struct { BuildInfo BuildInfo } -type ConfigSourceFactory interface { +type ConfigSourceFactoryBase interface { Factory CreateDefaultConfig() config.ConfigSource - CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (ConfigSource, error) +} + +type ConfigSourceFactory interface { + ConfigSourceFactoryBase + CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ConfigSource, error) } diff --git a/component/factories.go b/component/factories.go index 6682dff041c..654370e4409 100644 --- a/component/factories.go +++ b/component/factories.go @@ -35,7 +35,7 @@ type Factories struct { // Extensions maps extension type names in the config to the respective factory. Extensions map[config.Type]ExtensionFactory - ConfigSources map[config.Type]ConfigSourceFactory + ConfigSources map[config.Type]ConfigSourceFactoryBase } // MakeReceiverFactoryMap takes a list of receiver factories and returns a map diff --git a/config/configmapprovider/configsource.go b/config/configmapprovider/configsource.go new file mode 100644 index 00000000000..528f028f376 --- /dev/null +++ b/config/configmapprovider/configsource.go @@ -0,0 +1,65 @@ +package configmapprovider + +import ( + "context" + + "go.opentelemetry.io/collector/config" +) + +// ConfigSource is an interface that helps to retrieve a config map and watch for any +// changes to the config map. Implementations may load the config from a file, +// a database or any other source. +type ConfigSource interface { + Shutdownable + + // Retrieve goes to the configuration source and retrieves the selected data which + // contains the value to be injected in the configuration and the corresponding watcher that + // will be used to monitor for updates of the retrieved value. + // + // onChange callback is called when the config changes. onChange may be called from + // a different go routine. After onChange is called Retrieved.Get should be called + // to get the new config. See description of Retrieved for more details. + // onChange may be nil, which indicates that the caller is not interested in + // knowing about the changes. + // + // If ctx is cancelled should return immediately with an error. + // Should never be called concurrently with itself or with Shutdown. + Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) +} + +// RetrievedConfig holds the result of a call to the Retrieve method of a ConfigSource object. +// +// The typical usage is the following: +// +// r := mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// r = mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. +// // ... +// mapProvider.Shutdown() +type RetrievedConfig interface { + // Get returns the config Map. + // If Close is called before Get or concurrently with Get then Get + // should return immediately with ErrSessionClosed error. + // Should never be called concurrently with itself. + // If ctx is cancelled should return immediately with an error. + Get(ctx context.Context) (*config.Map, error) + + // Close signals that the configuration for which it was used to retrieve values is + // no longer in use and the object should close and release any watchers that it + // may have created. + // + // This method must be called when the service ends, either in case of success or error. + // + // Should never be called concurrently with itself. + // May be called before, after or concurrently with Get. + // If ctx is cancelled should return immediately with an error. + // + // Calling Close on an already closed object should have no effect and should return nil. + Close(ctx context.Context) error +} diff --git a/config/configmapprovider/default.go b/config/configmapprovider/default.go index ad62986e4da..80a469e8cbf 100644 --- a/config/configmapprovider/default.go +++ b/config/configmapprovider/default.go @@ -14,9 +14,9 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" -// NewLocal returns the default Provider, and it creates configuration from a file +// NewLocal returns the default Shutdownable, and it creates configuration from a file // defined by the given configFile and overwrites fields using properties. -func NewLocal(configFileName string, properties []string) Provider { +func NewLocal(configFileName string, properties []string) ConfigSource { return NewExpand( NewMerge( NewFile(configFileName), diff --git a/config/configmapprovider/expand.go b/config/configmapprovider/expand.go index dfe66dfba45..3311a263f23 100644 --- a/config/configmapprovider/expand.go +++ b/config/configmapprovider/expand.go @@ -21,18 +21,18 @@ import ( ) type expandMapProvider struct { - base Provider + base ConfigSource } -// NewExpand returns a Provider, that expands all environment variables for a -// config.Map provided by the given Provider. -func NewExpand(base Provider) Provider { +// NewExpand returns a Shutdownable, that expands all environment variables for a +// config.Map provided by the given Shutdownable. +func NewExpand(base ConfigSource) ConfigSource { return &expandMapProvider{ base: base, } } -func (emp *expandMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { +func (emp *expandMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { retr, err := emp.base.Retrieve(ctx, onChange) if err != nil { return nil, fmt.Errorf("failed to retrieve from base provider: %w", err) diff --git a/config/configmapprovider/file.go b/config/configmapprovider/file.go index fae7e7ad368..da4d8a26ae0 100644 --- a/config/configmapprovider/file.go +++ b/config/configmapprovider/file.go @@ -26,14 +26,14 @@ type fileMapProvider struct { fileName string } -// NewFile returns a new Provider that reads the configuration from the given file. -func NewFile(fileName string) Provider { +// NewFile returns a new Shutdownable that reads the configuration from the given file. +func NewFile(fileName string) ConfigSource { return &fileMapProvider{ fileName: fileName, } } -func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (Retrieved, error) { +func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (RetrievedConfig, error) { if fmp.fileName == "" { return nil, errors.New("config file not specified") } diff --git a/config/configmapprovider/inmemory.go b/config/configmapprovider/inmemory.go index 4a19ee594e5..006f22062f0 100644 --- a/config/configmapprovider/inmemory.go +++ b/config/configmapprovider/inmemory.go @@ -25,12 +25,12 @@ type inMemoryMapProvider struct { buf io.Reader } -// NewInMemory returns a new Provider that reads the configuration, from the provided buffer, as YAML. -func NewInMemory(buf io.Reader) Provider { +// NewInMemory returns a new Shutdownable that reads the configuration, from the provided buffer, as YAML. +func NewInMemory(buf io.Reader) ConfigSource { return &inMemoryMapProvider{buf: buf} } -func (inp *inMemoryMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { +func (inp *inMemoryMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { cfg, err := config.NewMapFromBuffer(inp.buf) if err != nil { return nil, err diff --git a/config/configmapprovider/merge.go b/config/configmapprovider/merge.go index 98b0b339686..a74c1a78957 100644 --- a/config/configmapprovider/merge.go +++ b/config/configmapprovider/merge.go @@ -25,17 +25,17 @@ import ( // TODO: Add support to "merge" watchable interface. type mergeMapProvider struct { - providers []Provider + providers []ConfigSource } -// NewMerge returns a Provider, that merges the result from multiple Provider. +// NewMerge returns a Shutdownable, that merges the result from multiple Shutdownable. // // The ConfigMaps are merged in the given order, by merging all of them in order into an initial empty map. -func NewMerge(ps ...Provider) Provider { +func NewMerge(ps ...ConfigSource) ConfigSource { return &mergeMapProvider{providers: ps} } -func (mp *mergeMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { +func (mp *mergeMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { retCfgMap := config.NewMap() for _, p := range mp.providers { retr, err := p.Retrieve(ctx, onChange) diff --git a/config/configmapprovider/properties.go b/config/configmapprovider/properties.go index 4cb13e82f3f..571f0751bb3 100644 --- a/config/configmapprovider/properties.go +++ b/config/configmapprovider/properties.go @@ -29,18 +29,18 @@ type propertiesMapProvider struct { properties []string } -// NewProperties returns a Provider, that provides a config.Map from the given properties. +// NewProperties returns a Shutdownable, that provides a config.Map from the given properties. // // Properties must follow the Java properties format, key-value list separated by equal sign with a "." // as key delimiter. // ["processors.batch.timeout=2s", "processors.batch/foo.timeout=3s"] -func NewProperties(properties []string) Provider { +func NewProperties(properties []string) ConfigSource { return &propertiesMapProvider{ properties: properties, } } -func (pmp *propertiesMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { +func (pmp *propertiesMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { if len(pmp.properties) == 0 { return &simpleRetrieved{confMap: config.NewMap()}, nil } diff --git a/config/configmapprovider/provider.go b/config/configmapprovider/provider.go index 49c79350ed9..42fc248be82 100644 --- a/config/configmapprovider/provider.go +++ b/config/configmapprovider/provider.go @@ -16,30 +16,14 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/config import ( "context" - - "go.opentelemetry.io/collector/config" ) -// Provider is an interface that helps to retrieve a config map and watch for any +// Shutdownable is an interface that helps to retrieve a config map and watch for any // changes to the config map. Implementations may load the config from a file, // a database or any other source. -type Provider interface { - // Retrieve goes to the configuration source and retrieves the selected data which - // contains the value to be injected in the configuration and the corresponding watcher that - // will be used to monitor for updates of the retrieved value. - // - // onChange callback is called when the config changes. onChange may be called from - // a different go routine. After onChange is called Retrieved.Get should be called - // to get the new config. See description of Retrieved for more details. - // onChange may be nil, which indicates that the caller is not interested in - // knowing about the changes. - // - // If ctx is cancelled should return immediately with an error. - // Should never be called concurrently with itself or with Shutdown. - Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) - - // Shutdown signals that the configuration for which this Provider was used to - // retrieve values is no longer in use and the Provider should close and release +type Shutdownable interface { + // Shutdown signals that the configuration for which this Shutdownable was used to + // retrieve values is no longer in use and the Shutdownable should close and release // any resources that it may have created. // // This method must be called when the Collector service ends, either in case of @@ -50,43 +34,6 @@ type Provider interface { Shutdown(ctx context.Context) error } -// Retrieved holds the result of a call to the Retrieve method of a Provider object. -// -// The typical usage is the following: -// -// r := mapProvider.Retrieve() -// r.Get() -// // wait for onChange() to be called. -// r.Close() -// r = mapProvider.Retrieve() -// r.Get() -// // wait for onChange() to be called. -// r.Close() -// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. -// // ... -// mapProvider.Shutdown() -type Retrieved interface { - // Get returns the config Map. - // If Close is called before Get or concurrently with Get then Get - // should return immediately with ErrSessionClosed error. - // Should never be called concurrently with itself. - // If ctx is cancelled should return immediately with an error. - Get(ctx context.Context) (*config.Map, error) - - // Close signals that the configuration for which it was used to retrieve values is - // no longer in use and the object should close and release any watchers that it - // may have created. - // - // This method must be called when the service ends, either in case of success or error. - // - // Should never be called concurrently with itself. - // May be called before, after or concurrently with Get. - // If ctx is cancelled should return immediately with an error. - // - // Calling Close on an already closed object should have no effect and should return nil. - Close(ctx context.Context) error -} - // ChangeEvent describes the particular change event that happened with the config. // TODO: see if this can be eliminated. type ChangeEvent struct { diff --git a/config/configmapprovider/simple.go b/config/configmapprovider/simple.go index 3069933233e..8f3035385d7 100644 --- a/config/configmapprovider/simple.go +++ b/config/configmapprovider/simple.go @@ -25,7 +25,7 @@ type simpleProvider struct { confMap *config.Map } -func (s simpleProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { +func (s simpleProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { return &simpleRetrieved{confMap: s.confMap}, nil } @@ -33,7 +33,7 @@ func (s simpleProvider) Shutdown(ctx context.Context) error { return nil } -func NewSimple(confMap *config.Map) Provider { +func NewSimple(confMap *config.Map) ConfigSource { return &simpleProvider{confMap: confMap} } diff --git a/config/configmapprovider/valuesource.go b/config/configmapprovider/valuesource.go new file mode 100644 index 00000000000..eb3576ae24f --- /dev/null +++ b/config/configmapprovider/valuesource.go @@ -0,0 +1,79 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" + +import ( + "context" + + "go.opentelemetry.io/collector/config" +) + +// ValueSource is an interface that helps to retrieve a config map and watch for any +// changes to the config map. Implementations may load the config from a file, +// a database or any other source. +type ValueSource interface { + Shutdownable + + // Retrieve goes to the configuration source and retrieves the selected data which + // contains the value to be injected in the configuration and the corresponding watcher that + // will be used to monitor for updates of the retrieved value. + // + // onChange callback is called when the config changes. onChange may be called from + // a different go routine. After onChange is called Retrieved.Get should be called + // to get the new config. See description of Retrieved for more details. + // onChange may be nil, which indicates that the caller is not interested in + // knowing about the changes. + // + // If ctx is cancelled should return immediately with an error. + // Should never be called concurrently with itself or with Shutdown. + Retrieve(ctx context.Context, onChange func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) +} + +// RetrievedValue holds the result of a call to the Retrieve method of a Shutdownable object. +// +// The typical usage is the following: +// +// r := mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// r = mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. +// // ... +// mapProvider.Shutdown() +type RetrievedValue interface { + // Get returns the config Map. + // If Close is called before Get or concurrently with Get then Get + // should return immediately with ErrSessionClosed error. + // Should never be called concurrently with itself. + // If ctx is cancelled should return immediately with an error. + Get(ctx context.Context) (interface{}, error) + + // Close signals that the configuration for which it was used to retrieve values is + // no longer in use and the object should close and release any watchers that it + // may have created. + // + // This method must be called when the service ends, either in case of success or error. + // + // Should never be called concurrently with itself. + // May be called before, after or concurrently with Get. + // If ctx is cancelled should return immediately with an error. + // + // Calling Close on an already closed object should have no effect and should return nil. + Close(ctx context.Context) error +} diff --git a/configsource/env/config.go b/configsource/env/config.go new file mode 100644 index 00000000000..cb4885e6dde --- /dev/null +++ b/configsource/env/config.go @@ -0,0 +1,10 @@ +package env + +import ( + "go.opentelemetry.io/collector/config" +) + +// Config defines configuration for logging exporter. +type Config struct { + config.ConfigSourceSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct +} diff --git a/configsource/env/configsource.go b/configsource/env/configsource.go new file mode 100644 index 00000000000..97a945b24db --- /dev/null +++ b/configsource/env/configsource.go @@ -0,0 +1,32 @@ +package env + +import ( + "context" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" +) + +type configSource struct { +} + +func (c configSource) Retrieve( + ctx context.Context, onChange func(*configmapprovider.ChangeEvent), +) (configmapprovider.RetrievedConfig, error) { + return &retrieved{}, nil +} + +func (c configSource) Shutdown(ctx context.Context) error { + return nil +} + +type retrieved struct { +} + +func (r retrieved) Get(ctx context.Context) (*config.Map, error) { + return config.NewMap(), nil +} + +func (r retrieved) Close(ctx context.Context) error { + return nil +} diff --git a/configsource/env/factory.go b/configsource/env/factory.go new file mode 100644 index 00000000000..df677ad2e16 --- /dev/null +++ b/configsource/env/factory.go @@ -0,0 +1,34 @@ +package env + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" + "go.opentelemetry.io/collector/internal/internalinterface" +) + +type factory struct { + internalinterface.BaseInternal +} + +func (f factory) Type() config.Type { + return "env" +} + +func (f factory) CreateDefaultConfig() config.ConfigSource { + return &Config{ + ConfigSourceSettings: config.NewConfigSourceSettings(config.NewComponentID(f.Type())), + } +} + +func (f factory) CreateConfigSource( + ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource, +) (configmapprovider.ConfigSource, error) { + return &configSource{}, nil +} + +func NewFactory() component.ConfigSourceFactory { + return &factory{} +} diff --git a/configsource/files/factory.go b/configsource/files/factory.go index 4e2a091c347..916dee38b89 100644 --- a/configsource/files/factory.go +++ b/configsource/files/factory.go @@ -25,7 +25,7 @@ func (f factory) CreateDefaultConfig() config.ConfigSource { func (f factory) CreateConfigSource( ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource, -) (component.ConfigSource, error) { +) (configmapprovider.ConfigSource, error) { return configmapprovider.NewFile(cfg.(*Config).Name), nil } diff --git a/service/command.go b/service/command.go index 0d21a4fd885..d35ef55894e 100644 --- a/service/command.go +++ b/service/command.go @@ -16,6 +16,7 @@ package service // import "go.opentelemetry.io/collector/service" import ( "github.com/spf13/cobra" + "go.opentelemetry.io/collector/service/defaultconfigprovider" ) // NewCommand constructs a new cobra.Command using the given Collector. @@ -28,7 +29,7 @@ func NewCommand(set CollectorSettings) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { if set.ConfigMapProvider == nil { //set.ConfigMapProvider = configmapprovider.NewLocal(getConfigFlag(), getSetFlag()) - set.ConfigMapProvider = NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) + set.ConfigMapProvider = defaultconfigprovider.NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) } col, err := New(set) if err != nil { diff --git a/service/config_watcher.go b/service/config_watcher.go index 43377fff23f..232b03bedd7 100644 --- a/service/config_watcher.go +++ b/service/config_watcher.go @@ -25,7 +25,7 @@ import ( type configWatcher struct { cfg *config.Config - ret configmapprovider.Retrieved + ret configmapprovider.RetrievedConfig watcher chan error } diff --git a/service/config_watcher_test.go b/service/config_watcher_test.go index de8821b720d..b68ae6d3b0d 100644 --- a/service/config_watcher_test.go +++ b/service/config_watcher_test.go @@ -78,7 +78,7 @@ func TestConfigWatcher(t *testing.T) { tests := []struct { name string - parserProvider configmapprovider.Provider + parserProvider configmapprovider.Shutdownable configUnmarshaler configunmarshaler.ConfigUnmarshaler expectNewErr bool expectWatchErr bool @@ -104,7 +104,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "watch_err", - parserProvider: func() configmapprovider.Provider { + parserProvider: func() configmapprovider.Shutdownable { ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) m, err := ret.Get(context.Background()) @@ -116,7 +116,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "close_err", - parserProvider: func() configmapprovider.Provider { + parserProvider: func() configmapprovider.Shutdownable { ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) m, err := ret.Get(context.Background()) @@ -128,7 +128,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "ok", - parserProvider: func() configmapprovider.Provider { + parserProvider: func() configmapprovider.Shutdownable { // Use errRetrieved with nil errors to have Watchable interface implemented. ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestConfigWatcherWhenClosed(t *testing.T) { factories, errF := componenttest.NopFactories() require.NoError(t, errF) set := CollectorSettings{ - ConfigMapProvider: func() configmapprovider.Provider { + ConfigMapProvider: func() configmapprovider.Shutdownable { // Use errRetrieved with nil errors to have Watchable interface implemented. ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) diff --git a/service/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go similarity index 70% rename from service/defaultconfigprovider.go rename to service/defaultconfigprovider/defaultconfigprovider.go index 7e747d65d6f..3e9e54800d9 100644 --- a/service/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package service // import "go.opentelemetry.io/collector/config/configmapprovider" +package defaultconfigprovider // import "go.opentelemetry.io/collector/config/configmapprovider" import ( "context" @@ -26,18 +26,18 @@ import ( ) type defaultConfigProvider struct { - provider configmapprovider.Provider - merged configmapprovider.Provider + initial configmapprovider.ConfigSource + merged configmapprovider.ConfigSource factories component.Factories } -func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.Provider { +func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.ConfigSource { localProvider := configmapprovider.NewLocal(configFileName, properties) - return &defaultConfigProvider{provider: localProvider, factories: factories} + return &defaultConfigProvider{initial: localProvider, factories: factories} } -func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(event *configmapprovider.ChangeEvent)) (configmapprovider.Retrieved, error) { - r, err := mp.provider.Retrieve(ctx, onChange) +func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(event *configmapprovider.ChangeEvent)) (configmapprovider.RetrievedConfig, error) { + r, err := mp.initial.Retrieve(ctx, onChange) if err != nil { return nil, err } @@ -51,17 +51,23 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve sources = append(sources, configmapprovider.NewSimple(rootMap)) mp.merged = configmapprovider.NewMerge(sources...) - return mp.merged.Retrieve(ctx, onChange) + retrieved, err := mp.merged.Retrieve(ctx, onChange) + if err != nil { + return nil, fmt.Errorf("cannot retrive the configuration: %w", err) + } + + return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved}, nil } -func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ([]configmapprovider.Provider, error) { - // Unmarshal the "config_sources" section of rootMap and create a Provider for each +func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ([]configmapprovider.ConfigSource, error) { + // Unmarshal the "config_sources" section of rootMap and create a Shutdownable for each // config source using the factories.ConfigSources. - var providers []configmapprovider.Provider + var providers []configmapprovider.ConfigSource type RootConfig struct { ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` + ValueSources []map[string]map[string]interface{} `mapstructure:"value_sources"` } var rootCfg RootConfig err := rootMap.Unmarshal(&rootCfg) @@ -71,18 +77,23 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon for _, sourceCfg := range rootCfg.ConfigSources { for sourceType, settings := range sourceCfg { - factory, ok := factories.ConfigSources[config.Type(sourceType)] + factoryBase, ok := factories.ConfigSources[config.Type(sourceType)] if !ok { return nil, fmt.Errorf("unknown source type %q", sourceType) } - cfg := factory.CreateDefaultConfig() + cfg := factoryBase.CreateDefaultConfig() cfg.SetIDName(sourceType) // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. if err := configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { - return nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) // errorUnmarshalError(extensionsKeyName, id, err) + return nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) + } + + factory, ok := factoryBase.(component.ConfigSourceFactory) + if !ok { + return nil, fmt.Errorf("config source %q does not implement ConfigSourceFactory", sourceType) } source, err := factory.CreateConfigSource(ctx, component.ConfigSourceCreateSettings{}, cfg) diff --git a/service/defaultconfigprovider/valuesourcesubstitutor.go b/service/defaultconfigprovider/valuesourcesubstitutor.go new file mode 100644 index 00000000000..8f2ece71b75 --- /dev/null +++ b/service/defaultconfigprovider/valuesourcesubstitutor.go @@ -0,0 +1,463 @@ +package defaultconfigprovider + +import ( + "bytes" + "context" + "fmt" + "net/url" + "os" + "strings" + + "github.com/spf13/cast" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" + "gopkg.in/yaml.v2" +) + +const ( + // configSourceNameDelimChar is the char used to terminate the name of config source + // when it is used to retrieve values to inject in the configuration + configSourceNameDelimChar = ':' + // expandPrefixChar is the char used to prefix strings that can be expanded, + // either environment variables or config sources. + expandPrefixChar = '$' + // typeAndNameSeparator is the separator that is used between type and name in type/name + // composite keys. + typeAndNameSeparator = '/' +) + +// private error types to help with testability +type ( + errUnknownConfigSource struct{ error } +) + +type valueSourceSubstitutor struct { + onChange func(event *configmapprovider.ChangeEvent) + retrieved configmapprovider.RetrievedConfig + valueSources map[string]configmapprovider.ValueSource +} + +func (vp *valueSourceSubstitutor) Get(ctx context.Context) (*config.Map, error) { + cfgMap, err := vp.retrieved.Get(ctx) + if err != nil { + return nil, err + } + + return vp.substitute(ctx, cfgMap) +} + +func (vp *valueSourceSubstitutor) substitute(ctx context.Context, cfgMap *config.Map) (*config.Map, error) { + for _, k := range cfgMap.AllKeys() { + val, err := vp.parseConfigValue(ctx, cfgMap.Get(k)) + if err != nil { + return nil, err + } + cfgMap.Set(k, val) + } + + return cfgMap, nil +} + +func (vp *valueSourceSubstitutor) Close(ctx context.Context) error { + return vp.retrieved.Close(ctx) +} + +// parseConfigValue takes the value of a "config node" and process it recursively. The processing consists +// in transforming invocations of config sources and/or environment variables into literal data that can be +// used directly from a `config.Map` object. +func (vp *valueSourceSubstitutor) parseConfigValue(ctx context.Context, value interface{}) (interface{}, error) { + switch v := value.(type) { + case string: + // Only if the value of the node is a string it can contain an env var or config source + // invocation that requires transformation. + return vp.parseStringValue(ctx, v) + case []interface{}: + // The value is of type []interface{} when an array is used in the configuration, YAML example: + // + // array0: + // - elem0 + // - elem1 + // array1: + // - entry: + // str: elem0 + // - entry: + // str: $tstcfgsrc:elem1 + // + // Both "array0" and "array1" are going to be leaf config nodes hitting this case. + nslice := make([]interface{}, 0, len(v)) + for _, vint := range v { + value, err := vp.parseConfigValue(ctx, vint) + if err != nil { + return nil, err + } + nslice = append(nslice, value) + } + return nslice, nil + case map[string]interface{}: + // The value is of type map[string]interface{} when an array in the configuration is populated with map + // elements. From the case above (for type []interface{}) each element of "array1" is going to hit the + // the current case block. + nmap := make(map[interface{}]interface{}, len(v)) + for k, vint := range v { + value, err := vp.parseConfigValue(ctx, vint) + if err != nil { + return nil, err + } + nmap[k] = value + } + return nmap, nil + default: + // All other literals (int, boolean, etc) can't be further expanded so just return them as they are. + return v, nil + } +} + +// parseStringValue transforms environment variables and config sources, if any are present, on +// the given string in the configuration into an object to be inserted into the resulting configuration. +func (vp *valueSourceSubstitutor) parseStringValue(ctx context.Context, s string) (interface{}, error) { + // Code based on os.Expand function. All delimiters that are checked against are + // ASCII so bytes are fine for this operation. + var buf []byte + + // Using i, j, and w variables to keep correspondence with os.Expand code. + // i tracks the index in s from which a slice to be appended to buf should start. + // j tracks the char being currently checked and also the end of the slice to be appended to buf. + // w tracks the number of characters being consumed after a prefix identifying env vars or config sources. + i := 0 + for j := 0; j < len(s); j++ { + // Skip chars until a candidate for expansion is found. + if s[j] == expandPrefixChar && j+1 < len(s) { + if buf == nil { + // Assuming that the length of the string will double after expansion of env vars and config sources. + buf = make([]byte, 0, 2*len(s)) + } + + // Append everything consumed up to the prefix char (but not including the prefix char) to the result. + buf = append(buf, s[i:j]...) + + var expandableContent, cfgSrcName string + w := 0 // number of bytes consumed on this pass + + switch { + case s[j+1] == expandPrefixChar: + // Escaping the prefix so $$ becomes a single $ without attempting + // to treat the string after it as a config source or env var. + expandableContent = string(expandPrefixChar) + w = 1 // consumed a single char + + case s[j+1] == '{': + // Bracketed usage, consume everything until first '}' exactly as os.Expand. + expandableContent, w = scanToClosingBracket(s[j+1:]) + expandableContent = strings.Trim(expandableContent, " ") // Allow for some spaces. + delimIndex := strings.Index(expandableContent, string(configSourceNameDelimChar)) + if len(expandableContent) > 1 && delimIndex > -1 { + // Bracket expandableContent contains ':' treating it as a config source. + cfgSrcName = expandableContent[:delimIndex] + } + + default: + // Non-bracketed usage, ie.: found the prefix char, it can be either a config + // source or an environment variable. + var name string + name, w = getTokenName(s[j+1:]) + expandableContent = name // Assume for now that it is an env var. + + // Peek next char after name, if it is a config source name delimiter treat the remaining of the + // string as a config source. + possibleDelimCharIndex := j + w + 1 + if possibleDelimCharIndex < len(s) && s[possibleDelimCharIndex] == configSourceNameDelimChar { + // This is a config source, since it is not delimited it will consume until end of the string. + cfgSrcName = name + expandableContent = s[j+1:] + w = len(expandableContent) // Set consumed bytes to the length of expandableContent + } + } + + // At this point expandableContent contains a string to be expanded, evaluate and expand it. + switch { + case cfgSrcName == "": + // Not a config source, expand as os.ExpandEnv + buf = osExpandEnv(buf, expandableContent, w) + + default: + // A config source, retrieve and apply results. + retrieved, err := vp.retrieveConfigSourceData(ctx, cfgSrcName, expandableContent) + if err != nil { + return nil, err + } + + consumedAll := j+w+1 == len(s) + if consumedAll && len(buf) == 0 { + // This is the only expandableContent on the string, config + // source is free to return interface{} but parse it as YAML + // if it is a string or byte slice. + switch value := retrieved.(type) { + case []byte: + if err := yaml.Unmarshal(value, &retrieved); err != nil { + // The byte slice is an invalid YAML keep the original. + retrieved = value + } + case string: + if err := yaml.Unmarshal([]byte(value), &retrieved); err != nil { + // The string is an invalid YAML keep it as the original. + retrieved = value + } + } + + if mapIFace, ok := retrieved.(map[interface{}]interface{}); ok { + // yaml.Unmarshal returns map[interface{}]interface{} but config + // map uses map[string]interface{}, fix it with a cast. + retrieved = cast.ToStringMap(mapIFace) + } + + return retrieved, nil + } + + // Either there was a prefix already or there are still characters to be processed. + if retrieved == nil { + // Since this is going to be concatenated to a string use "" instead of nil, + // otherwise the string will end up with "". + retrieved = "" + } + + buf = append(buf, fmt.Sprintf("%v", retrieved)...) + } + + j += w // move the index of the char being checked (j) by the number of characters consumed (w) on this iteration. + i = j + 1 // update start index (i) of next slice of bytes to be copied. + } + } + + if buf == nil { + // No changes to original string, just return it. + return s, nil + } + + // Return whatever was accumulated on the buffer plus the remaining of the original string. + return string(buf) + s[i:], nil +} + +// retrieveConfigSourceData retrieves data from the specified config source and injects them into +// the configuration. The Manager tracks sessions and watcher objects as needed. +func (vp *valueSourceSubstitutor) retrieveConfigSourceData(ctx context.Context, cfgSrcName, cfgSrcInvocation string) (interface{}, error) { + valueSrc, ok := vp.valueSources[cfgSrcName] + if !ok { + return nil, newErrUnknownConfigSource(cfgSrcName) + } + + cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(cfgSrcInvocation) + if err != nil { + return nil, err + } + + // Recursively expand the selector. + var expandedSelector interface{} + expandedSelector, err = vp.parseStringValue(ctx, selector) + if err != nil { + return nil, fmt.Errorf("failed to process selector for config source %q selector %q: %w", cfgSrcName, selector, err) + } + if selector, ok = expandedSelector.(string); !ok { + return nil, fmt.Errorf("processed selector must be a string instead got a %T %v", expandedSelector, expandedSelector) + } + + // Recursively resolve/parse any config source on the parameters. + if paramsConfigMap != nil { + paramsConfigMap, err = vp.substitute(ctx, paramsConfigMap) + if err != nil { + return nil, fmt.Errorf("failed to process parameters for config source %q invocation %q: %w", cfgSrcName, cfgSrcInvocation, err) + } + } + + retrieved, err := valueSrc.Retrieve(ctx, vp.onChange, selector, paramsConfigMap) + if err != nil { + return nil, fmt.Errorf("config source %q failed to retrieve value: %w", cfgSrcName, err) + } + + valMap, err := retrieved.Get(ctx) + if err != nil { + return nil, err + } + + return valMap, nil +} + +func newErrUnknownConfigSource(cfgSrcName string) error { + return &errUnknownConfigSource{ + fmt.Errorf(`config source %q not found if this was intended to be an environment variable use "${%s}" instead"`, cfgSrcName, cfgSrcName), + } +} + +// parseCfgSrcInvocation parses the original string in the configuration that has a config source +// retrieve operation and return its "logical components": the config source name, the selector, and +// a config.Map to be used in this invocation of the config source. See Test_parseCfgSrcInvocation +// for some examples of input and output. +// The caller should check for error explicitly since it is possible for the +// other values to have been partially set. +func parseCfgSrcInvocation(s string) (cfgSrcName, selector string, paramsConfigMap *config.Map, err error) { + parts := strings.SplitN(s, string(configSourceNameDelimChar), 2) + if len(parts) != 2 { + err = fmt.Errorf("invalid config source syntax at %q, it must have at least the config source name and a selector", s) + return + } + cfgSrcName = strings.Trim(parts[0], " ") + + // Separate multi-line and single line case. + afterCfgSrcName := parts[1] + switch { + case strings.Contains(afterCfgSrcName, "\n"): + // Multi-line, until the first \n it is the selector, everything after as YAML. + parts = strings.SplitN(afterCfgSrcName, "\n", 2) + selector = strings.Trim(parts[0], " ") + + if len(parts) > 1 && len(parts[1]) > 0 { + var cp *config.Map + cp, err = config.NewMapFromBuffer(bytes.NewReader([]byte(parts[1]))) + if err != nil { + return + } + paramsConfigMap = cp + } + + default: + // Single line, and parameters as URL query. + const selectorDelim string = "?" + parts = strings.SplitN(parts[1], selectorDelim, 2) + selector = strings.Trim(parts[0], " ") + + if len(parts) == 2 { + paramsPart := parts[1] + paramsConfigMap, err = parseParamsAsURLQuery(paramsPart) + if err != nil { + err = fmt.Errorf("invalid parameters syntax at %q: %w", s, err) + return + } + } + } + + return cfgSrcName, selector, paramsConfigMap, err +} + +func parseParamsAsURLQuery(s string) (*config.Map, error) { + values, err := url.ParseQuery(s) + if err != nil { + return nil, err + } + + // Transform single array values in scalars. + params := make(map[string]interface{}) + for k, v := range values { + switch len(v) { + case 0: + params[k] = nil + case 1: + var iface interface{} + if err = yaml.Unmarshal([]byte(v[0]), &iface); err != nil { + return nil, err + } + params[k] = iface + default: + // It is a slice add element by element + elemSlice := make([]interface{}, 0, len(v)) + for _, elem := range v { + var iface interface{} + if err = yaml.Unmarshal([]byte(elem), &iface); err != nil { + return nil, err + } + elemSlice = append(elemSlice, iface) + } + params[k] = elemSlice + } + } + return config.NewMapFromStringMap(params), err +} + +// osExpandEnv replicate the internal behavior of os.ExpandEnv when handling env +// vars updating the buffer accordingly. +func osExpandEnv(buf []byte, name string, w int) []byte { + switch { + case name == "" && w > 0: + // Encountered invalid syntax; eat the + // characters. + case name == "" || name == "$": + // Valid syntax, but $ was not followed by a + // name. Leave the dollar character untouched. + buf = append(buf, expandPrefixChar) + default: + buf = append(buf, os.Getenv(name)...) + } + + return buf +} + +// scanToClosingBracket consumes everything until a closing bracket '}' following the +// same logic of function getShellName (os package, env.go) when handling environment +// variables with the "${}" syntax. It returns the expression between brackets +// and the number of characters consumed from the original string. +func scanToClosingBracket(s string) (string, int) { + for i := 1; i < len(s); i++ { + if s[i] == '}' { + if i == 1 { + return "", 2 // Bad syntax; eat "${}" + } + return s[1:i], i + 1 + } + } + return "", 1 // Bad syntax; eat "${" +} + +// getTokenName consumes characters until it has the name of either an environment +// variable or config source. It returns the name of the config source or environment +// variable and the number of characters consumed from the original string. +func getTokenName(s string) (string, int) { + if len(s) > 0 && isShellSpecialVar(s[0]) { + // Special shell character, treat it os.Expand function. + return s[0:1], 1 + } + + var i int + firstNameSepIdx := -1 + for i = 0; i < len(s); i++ { + if isAlphaNum(s[i]) { + // Continue while alphanumeric plus underscore. + continue + } + + if s[i] == typeAndNameSeparator && firstNameSepIdx == -1 { + // If this is the first type name separator store the index and continue. + firstNameSepIdx = i + continue + } + + // It is one of the following cases: + // 1. End of string + // 2. Reached a non-alphanumeric character, preceded by at most one + // typeAndNameSeparator character. + break + } + + if firstNameSepIdx != -1 && (i >= len(s) || s[i] != configSourceNameDelimChar) { + // Found a second non alpha-numeric character before the end of the string + // but it is not the config source delimiter. Use the name until the first + // name delimiter. + return s[:firstNameSepIdx], firstNameSepIdx + } + + return s[:i], i +} + +// Below are helper functions used by os.Expand, copied without changes from original sources (env.go). + +// isShellSpecialVar reports whether the character identifies a special +// shell variable such as $*. +func isShellSpecialVar(c uint8) bool { + switch c { + case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return true + } + return false +} + +// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore +func isAlphaNum(c uint8) bool { + return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' +} diff --git a/service/settings.go b/service/settings.go index 59a16de4fc5..c945a720107 100644 --- a/service/settings.go +++ b/service/settings.go @@ -64,7 +64,7 @@ type CollectorSettings struct { // from a config file define by the --config command line flag and overrides component's configuration // properties supplied via --set command line flag. // If the provider is configmapprovider.WatchableRetrieved, collector may reload the configuration upon error. - ConfigMapProvider configmapprovider.Provider + ConfigMapProvider configmapprovider.ConfigSource // ConfigUnmarshaler unmarshalls the configuration's Parser into the service configuration. // If it is not provided a default unmarshaler is used. From c3fc4354b55599280aec6643d2cc800fb1935f0f Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Fri, 19 Nov 2021 17:44:25 -0500 Subject: [PATCH 03/23] Unmarshal value sources and use --- component/configsource.go | 5 ++ configsource/env/configsource.go | 13 ++-- configsource/env/factory.go | 6 +- .../defaultconfigprovider.go | 67 ++++++++++++++----- .../valuesourcesubstitutor.go | 11 ++- 5 files changed, 74 insertions(+), 28 deletions(-) diff --git a/component/configsource.go b/component/configsource.go index e6bcce4cef4..bee20ac37a6 100644 --- a/component/configsource.go +++ b/component/configsource.go @@ -43,3 +43,8 @@ type ConfigSourceFactory interface { ConfigSourceFactoryBase CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ConfigSource, error) } + +type ValueSourceFactory interface { + ConfigSourceFactoryBase + CreateValueSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ValueSource, error) +} diff --git a/configsource/env/configsource.go b/configsource/env/configsource.go index 97a945b24db..5c5218f0b67 100644 --- a/configsource/env/configsource.go +++ b/configsource/env/configsource.go @@ -2,6 +2,7 @@ package env import ( "context" + "os" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/configmapprovider" @@ -11,9 +12,9 @@ type configSource struct { } func (c configSource) Retrieve( - ctx context.Context, onChange func(*configmapprovider.ChangeEvent), -) (configmapprovider.RetrievedConfig, error) { - return &retrieved{}, nil + ctx context.Context, onChange func(*configmapprovider.ChangeEvent), selector string, paramsConfigMap *config.Map, +) (configmapprovider.RetrievedValue, error) { + return &retrieved{selector: selector, paramsConfigMap: paramsConfigMap}, nil } func (c configSource) Shutdown(ctx context.Context) error { @@ -21,10 +22,12 @@ func (c configSource) Shutdown(ctx context.Context) error { } type retrieved struct { + selector string + paramsConfigMap *config.Map } -func (r retrieved) Get(ctx context.Context) (*config.Map, error) { - return config.NewMap(), nil +func (r retrieved) Get(ctx context.Context) (interface{}, error) { + return os.Getenv(r.selector), nil } func (r retrieved) Close(ctx context.Context) error { diff --git a/configsource/env/factory.go b/configsource/env/factory.go index df677ad2e16..0ccbb7ca06b 100644 --- a/configsource/env/factory.go +++ b/configsource/env/factory.go @@ -23,12 +23,10 @@ func (f factory) CreateDefaultConfig() config.ConfigSource { } } -func (f factory) CreateConfigSource( - ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource, -) (configmapprovider.ConfigSource, error) { +func (f factory) CreateValueSource(ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ValueSource, error) { return &configSource{}, nil } -func NewFactory() component.ConfigSourceFactory { +func NewFactory() component.ValueSourceFactory { return &factory{} } diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 3e9e54800d9..abd1c91e822 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -43,43 +43,45 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve } rootMap, err := r.Get(ctx) - sources, err := unmarshalSources(ctx, rootMap, mp.factories) + configSources, valueSources, err := unmarshalSources(ctx, rootMap, mp.factories) if err != nil { return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) } - sources = append(sources, configmapprovider.NewSimple(rootMap)) - mp.merged = configmapprovider.NewMerge(sources...) + configSources = append(configSources, configmapprovider.NewSimple(rootMap)) + mp.merged = configmapprovider.NewMerge(configSources...) retrieved, err := mp.merged.Retrieve(ctx, onChange) if err != nil { return nil, fmt.Errorf("cannot retrive the configuration: %w", err) } - return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved}, nil + return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved, valueSources: valueSources}, nil } -func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ([]configmapprovider.ConfigSource, error) { +func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( + configSources []configmapprovider.ConfigSource, + valueSources map[config.ComponentID]configmapprovider.ValueSource, + err error, +) { // Unmarshal the "config_sources" section of rootMap and create a Shutdownable for each // config source using the factories.ConfigSources. - var providers []configmapprovider.ConfigSource - type RootConfig struct { ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` - ValueSources []map[string]map[string]interface{} `mapstructure:"value_sources"` + ValueSources map[string]map[string]interface{} `mapstructure:"value_sources"` } var rootCfg RootConfig - err := rootMap.Unmarshal(&rootCfg) + err = rootMap.Unmarshal(&rootCfg) if err != nil { - return nil, err + return nil, nil, err } for _, sourceCfg := range rootCfg.ConfigSources { for sourceType, settings := range sourceCfg { factoryBase, ok := factories.ConfigSources[config.Type(sourceType)] if !ok { - return nil, fmt.Errorf("unknown source type %q", sourceType) + return nil, nil, fmt.Errorf("unknown source type %q", sourceType) } cfg := factoryBase.CreateDefaultConfig() @@ -88,23 +90,56 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. if err := configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { - return nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) + return nil, nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) } factory, ok := factoryBase.(component.ConfigSourceFactory) if !ok { - return nil, fmt.Errorf("config source %q does not implement ConfigSourceFactory", sourceType) + return nil, nil, fmt.Errorf("config source %q does not implement ConfigSourceFactory", sourceType) } source, err := factory.CreateConfigSource(ctx, component.ConfigSourceCreateSettings{}, cfg) if err != nil { - return nil, err + return nil, nil, err } - providers = append(providers, source) + configSources = append(configSources, source) + } + } + + valueSources = map[config.ComponentID]configmapprovider.ValueSource{} + for sourceName, settings := range rootCfg.ValueSources { + id, err := config.NewComponentIDFromString(sourceName) + if err != nil { + return nil, nil, err + } + sourceType := id.Type() + factoryBase, ok := factories.ConfigSources[sourceType] + if !ok { + return nil, nil, fmt.Errorf("unknown source type %q", sourceType) + } + + cfg := factoryBase.CreateDefaultConfig() + cfg.SetIDName(id.Name()) + + // Now that the default config struct is created we can Unmarshal into it, + // and it will apply user-defined config on top of the default. + if err := configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { + return nil, nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) + } + + factory, ok := factoryBase.(component.ValueSourceFactory) + if !ok { + return nil, nil, fmt.Errorf("config source %q does not implement ValueSourceFactory", sourceType) + } + + source, err := factory.CreateValueSource(ctx, component.ConfigSourceCreateSettings{}, cfg) + if err != nil { + return nil, nil, err } + valueSources[cfg.ID()] = source } - return providers, nil + return configSources, valueSources, nil } func (mp *defaultConfigProvider) Shutdown(ctx context.Context) error { diff --git a/service/defaultconfigprovider/valuesourcesubstitutor.go b/service/defaultconfigprovider/valuesourcesubstitutor.go index 8f2ece71b75..1ae80d6987b 100644 --- a/service/defaultconfigprovider/valuesourcesubstitutor.go +++ b/service/defaultconfigprovider/valuesourcesubstitutor.go @@ -34,7 +34,7 @@ type ( type valueSourceSubstitutor struct { onChange func(event *configmapprovider.ChangeEvent) retrieved configmapprovider.RetrievedConfig - valueSources map[string]configmapprovider.ValueSource + valueSources map[config.ComponentID]configmapprovider.ValueSource } func (vp *valueSourceSubstitutor) Get(ctx context.Context) (*config.Map, error) { @@ -239,8 +239,13 @@ func (vp *valueSourceSubstitutor) parseStringValue(ctx context.Context, s string // retrieveConfigSourceData retrieves data from the specified config source and injects them into // the configuration. The Manager tracks sessions and watcher objects as needed. -func (vp *valueSourceSubstitutor) retrieveConfigSourceData(ctx context.Context, cfgSrcName, cfgSrcInvocation string) (interface{}, error) { - valueSrc, ok := vp.valueSources[cfgSrcName] +func (vp *valueSourceSubstitutor) retrieveConfigSourceData(ctx context.Context, cfgSrcName string, cfgSrcInvocation string) (interface{}, error) { + cfgSrcID, err := config.NewComponentIDFromString(cfgSrcName) + if err != nil { + return nil, err + } + + valueSrc, ok := vp.valueSources[cfgSrcID] if !ok { return nil, newErrUnknownConfigSource(cfgSrcName) } From d4a178e0445b0fb4d58505d348268b65b28091f0 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:19:39 -0500 Subject: [PATCH 04/23] Rearrange root config structure Use "merge_configs" to specify root config sources to merge --- component/configsource.go | 17 +-- component/factories.go | 2 +- config/configmapprovider/baseprovider.go | 44 ++++++++ config/configmapprovider/configsource.go | 65 ----------- config/configmapprovider/default.go | 9 +- config/configmapprovider/expand.go | 11 +- config/configmapprovider/file.go | 2 +- config/configmapprovider/inmemory.go | 2 +- config/configmapprovider/merge.go | 6 +- config/configmapprovider/properties.go | 2 +- config/configmapprovider/provider.go | 83 ++++++++------ config/configmapprovider/simple.go | 2 +- .../{valuesource.go => valueprovider.go} | 6 +- .../configunmarshaler/defaultunmarshaler.go | 2 +- configsource/env/factory.go | 4 +- configsource/files/factory.go | 2 +- service/config_watcher.go | 21 ---- .../defaultconfigprovider.go | 104 +++++++++--------- .../valuesourcesubstitutor.go | 2 +- service/settings.go | 2 +- 20 files changed, 182 insertions(+), 206 deletions(-) create mode 100644 config/configmapprovider/baseprovider.go delete mode 100644 config/configmapprovider/configsource.go rename config/configmapprovider/{valuesource.go => valueprovider.go} (95%) diff --git a/component/configsource.go b/component/configsource.go index bee20ac37a6..a2a4520ae42 100644 --- a/component/configsource.go +++ b/component/configsource.go @@ -23,28 +23,17 @@ import ( // ConfigSource is type ConfigSource interface { - configmapprovider.Shutdownable + configmapprovider.BaseProvider } // ConfigSourceCreateSettings is passed to ExtensionFactory.Create* functions. type ConfigSourceCreateSettings struct { - //TelemetrySettings - // BuildInfo can be used by components for informational purposes BuildInfo BuildInfo } -type ConfigSourceFactoryBase interface { +type ConfigSourceFactory interface { Factory CreateDefaultConfig() config.ConfigSource -} - -type ConfigSourceFactory interface { - ConfigSourceFactoryBase - CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ConfigSource, error) -} - -type ValueSourceFactory interface { - ConfigSourceFactoryBase - CreateValueSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ValueSource, error) + CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.BaseProvider, error) } diff --git a/component/factories.go b/component/factories.go index 654370e4409..6682dff041c 100644 --- a/component/factories.go +++ b/component/factories.go @@ -35,7 +35,7 @@ type Factories struct { // Extensions maps extension type names in the config to the respective factory. Extensions map[config.Type]ExtensionFactory - ConfigSources map[config.Type]ConfigSourceFactoryBase + ConfigSources map[config.Type]ConfigSourceFactory } // MakeReceiverFactoryMap takes a list of receiver factories and returns a map diff --git a/config/configmapprovider/baseprovider.go b/config/configmapprovider/baseprovider.go new file mode 100644 index 00000000000..fcb2782ffc7 --- /dev/null +++ b/config/configmapprovider/baseprovider.go @@ -0,0 +1,44 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" + +import ( + "context" +) + +// BaseProvider is an interface that helps to retrieve a config map and watch for any +// changes to the config map. Implementations may load the config from a file, +// a database or any other source. +type BaseProvider interface { + // Shutdown signals that the configuration for which this Shutdownable was used to + // retrieve values is no longer in use and the Shutdownable should close and release + // any resources that it may have created. + // + // This method must be called when the Collector service ends, either in case of + // success or error. Retrieve cannot be called after Shutdown. + // + // Should never be called concurrently with itself or with Retrieve. + // If ctx is cancelled should return immediately with an error. + Shutdown(ctx context.Context) error +} + +// ChangeEvent describes the particular change event that happened with the config. +// TODO: see if this can be eliminated. +type ChangeEvent struct { + // Error is nil if the config is changed and needs to be re-fetched using Get. + // It is set to configsource.ErrSessionClosed if Close was called. + // Any other error indicates that there was a problem with retrieving the config. + Error error +} diff --git a/config/configmapprovider/configsource.go b/config/configmapprovider/configsource.go deleted file mode 100644 index 528f028f376..00000000000 --- a/config/configmapprovider/configsource.go +++ /dev/null @@ -1,65 +0,0 @@ -package configmapprovider - -import ( - "context" - - "go.opentelemetry.io/collector/config" -) - -// ConfigSource is an interface that helps to retrieve a config map and watch for any -// changes to the config map. Implementations may load the config from a file, -// a database or any other source. -type ConfigSource interface { - Shutdownable - - // Retrieve goes to the configuration source and retrieves the selected data which - // contains the value to be injected in the configuration and the corresponding watcher that - // will be used to monitor for updates of the retrieved value. - // - // onChange callback is called when the config changes. onChange may be called from - // a different go routine. After onChange is called Retrieved.Get should be called - // to get the new config. See description of Retrieved for more details. - // onChange may be nil, which indicates that the caller is not interested in - // knowing about the changes. - // - // If ctx is cancelled should return immediately with an error. - // Should never be called concurrently with itself or with Shutdown. - Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) -} - -// RetrievedConfig holds the result of a call to the Retrieve method of a ConfigSource object. -// -// The typical usage is the following: -// -// r := mapProvider.Retrieve() -// r.Get() -// // wait for onChange() to be called. -// r.Close() -// r = mapProvider.Retrieve() -// r.Get() -// // wait for onChange() to be called. -// r.Close() -// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. -// // ... -// mapProvider.Shutdown() -type RetrievedConfig interface { - // Get returns the config Map. - // If Close is called before Get or concurrently with Get then Get - // should return immediately with ErrSessionClosed error. - // Should never be called concurrently with itself. - // If ctx is cancelled should return immediately with an error. - Get(ctx context.Context) (*config.Map, error) - - // Close signals that the configuration for which it was used to retrieve values is - // no longer in use and the object should close and release any watchers that it - // may have created. - // - // This method must be called when the service ends, either in case of success or error. - // - // Should never be called concurrently with itself. - // May be called before, after or concurrently with Get. - // If ctx is cancelled should return immediately with an error. - // - // Calling Close on an already closed object should have no effect and should return nil. - Close(ctx context.Context) error -} diff --git a/config/configmapprovider/default.go b/config/configmapprovider/default.go index 80a469e8cbf..1ae8f2288fd 100644 --- a/config/configmapprovider/default.go +++ b/config/configmapprovider/default.go @@ -16,9 +16,8 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/config // NewLocal returns the default Shutdownable, and it creates configuration from a file // defined by the given configFile and overwrites fields using properties. -func NewLocal(configFileName string, properties []string) ConfigSource { - return NewExpand( - NewMerge( - NewFile(configFileName), - NewProperties(properties))) +func NewLocal(configFileName string, properties []string) Provider { + return NewMerge( + NewFile(configFileName), + NewProperties(properties)) } diff --git a/config/configmapprovider/expand.go b/config/configmapprovider/expand.go index 3311a263f23..0824b4d87e4 100644 --- a/config/configmapprovider/expand.go +++ b/config/configmapprovider/expand.go @@ -18,15 +18,16 @@ import ( "context" "fmt" "os" + "strings" ) type expandMapProvider struct { - base ConfigSource + base Provider } // NewExpand returns a Shutdownable, that expands all environment variables for a // config.Map provided by the given Shutdownable. -func NewExpand(base ConfigSource) ConfigSource { +func NewExpand(base Provider) Provider { return &expandMapProvider{ base: base, } @@ -79,6 +80,12 @@ func expandStringValues(value interface{}) interface{} { } func expandEnv(s string) string { + if strings.Contains(s, ":") { + // This uses the extended syntax ${valuesrc:selector} which will be expanded + // later by ValueProvider substitutor, so don't touch it. + return s + } + return os.Expand(s, func(str string) string { // This allows escaping environment variable substitution via $$, e.g. // - $FOO will be substituted with env var FOO diff --git a/config/configmapprovider/file.go b/config/configmapprovider/file.go index da4d8a26ae0..efbd5efeaa8 100644 --- a/config/configmapprovider/file.go +++ b/config/configmapprovider/file.go @@ -27,7 +27,7 @@ type fileMapProvider struct { } // NewFile returns a new Shutdownable that reads the configuration from the given file. -func NewFile(fileName string) ConfigSource { +func NewFile(fileName string) Provider { return &fileMapProvider{ fileName: fileName, } diff --git a/config/configmapprovider/inmemory.go b/config/configmapprovider/inmemory.go index 006f22062f0..43d6ab9f27d 100644 --- a/config/configmapprovider/inmemory.go +++ b/config/configmapprovider/inmemory.go @@ -26,7 +26,7 @@ type inMemoryMapProvider struct { } // NewInMemory returns a new Shutdownable that reads the configuration, from the provided buffer, as YAML. -func NewInMemory(buf io.Reader) ConfigSource { +func NewInMemory(buf io.Reader) BaseProvider { return &inMemoryMapProvider{buf: buf} } diff --git a/config/configmapprovider/merge.go b/config/configmapprovider/merge.go index a74c1a78957..ff68fed2ede 100644 --- a/config/configmapprovider/merge.go +++ b/config/configmapprovider/merge.go @@ -25,13 +25,13 @@ import ( // TODO: Add support to "merge" watchable interface. type mergeMapProvider struct { - providers []ConfigSource + providers []Provider } -// NewMerge returns a Shutdownable, that merges the result from multiple Shutdownable. +// NewMerge returns a Provider, that merges the result from multiple Provider(s). // // The ConfigMaps are merged in the given order, by merging all of them in order into an initial empty map. -func NewMerge(ps ...ConfigSource) ConfigSource { +func NewMerge(ps ...Provider) Provider { return &mergeMapProvider{providers: ps} } diff --git a/config/configmapprovider/properties.go b/config/configmapprovider/properties.go index 571f0751bb3..1c0fea0c347 100644 --- a/config/configmapprovider/properties.go +++ b/config/configmapprovider/properties.go @@ -34,7 +34,7 @@ type propertiesMapProvider struct { // Properties must follow the Java properties format, key-value list separated by equal sign with a "." // as key delimiter. // ["processors.batch.timeout=2s", "processors.batch/foo.timeout=3s"] -func NewProperties(properties []string) ConfigSource { +func NewProperties(properties []string) Provider { return &propertiesMapProvider{ properties: properties, } diff --git a/config/configmapprovider/provider.go b/config/configmapprovider/provider.go index 42fc248be82..f3aafc34ba7 100644 --- a/config/configmapprovider/provider.go +++ b/config/configmapprovider/provider.go @@ -1,44 +1,65 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" +package configmapprovider import ( "context" + + "go.opentelemetry.io/collector/config" ) -// Shutdownable is an interface that helps to retrieve a config map and watch for any +// Provider is an interface that helps to retrieve a config map and watch for any // changes to the config map. Implementations may load the config from a file, // a database or any other source. -type Shutdownable interface { - // Shutdown signals that the configuration for which this Shutdownable was used to - // retrieve values is no longer in use and the Shutdownable should close and release - // any resources that it may have created. +type Provider interface { + BaseProvider + + // Retrieve goes to the configuration source and retrieves the selected data which + // contains the value to be injected in the configuration and the corresponding watcher that + // will be used to monitor for updates of the retrieved value. // - // This method must be called when the Collector service ends, either in case of - // success or error. Retrieve cannot be called after Shutdown. + // onChange callback is called when the config changes. onChange may be called from + // a different go routine. After onChange is called Retrieved.Get should be called + // to get the new config. See description of Retrieved for more details. + // onChange may be nil, which indicates that the caller is not interested in + // knowing about the changes. // - // Should never be called concurrently with itself or with Retrieve. // If ctx is cancelled should return immediately with an error. - Shutdown(ctx context.Context) error + // Should never be called concurrently with itself or with Shutdown. + Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) } -// ChangeEvent describes the particular change event that happened with the config. -// TODO: see if this can be eliminated. -type ChangeEvent struct { - // Error is nil if the config is changed and needs to be re-fetched using Get. - // It is set to configsource.ErrSessionClosed if Close was called. - // Any other error indicates that there was a problem with retrieving the config. - Error error +// RetrievedConfig holds the result of a call to the Retrieve method of a BaseProvider object. +// +// The typical usage is the following: +// +// r := mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// r = mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. +// // ... +// mapProvider.Shutdown() +type RetrievedConfig interface { + // Get returns the config Map. + // If Close is called before Get or concurrently with Get then Get + // should return immediately with ErrSessionClosed error. + // Should never be called concurrently with itself. + // If ctx is cancelled should return immediately with an error. + Get(ctx context.Context) (*config.Map, error) + + // Close signals that the configuration for which it was used to retrieve values is + // no longer in use and the object should close and release any watchers that it + // may have created. + // + // This method must be called when the service ends, either in case of success or error. + // + // Should never be called concurrently with itself. + // May be called before, after or concurrently with Get. + // If ctx is cancelled should return immediately with an error. + // + // Calling Close on an already closed object should have no effect and should return nil. + Close(ctx context.Context) error } diff --git a/config/configmapprovider/simple.go b/config/configmapprovider/simple.go index 8f3035385d7..7fb4fa2192e 100644 --- a/config/configmapprovider/simple.go +++ b/config/configmapprovider/simple.go @@ -33,7 +33,7 @@ func (s simpleProvider) Shutdown(ctx context.Context) error { return nil } -func NewSimple(confMap *config.Map) ConfigSource { +func NewSimple(confMap *config.Map) Provider { return &simpleProvider{confMap: confMap} } diff --git a/config/configmapprovider/valuesource.go b/config/configmapprovider/valueprovider.go similarity index 95% rename from config/configmapprovider/valuesource.go rename to config/configmapprovider/valueprovider.go index eb3576ae24f..027596f70a2 100644 --- a/config/configmapprovider/valuesource.go +++ b/config/configmapprovider/valueprovider.go @@ -20,11 +20,11 @@ import ( "go.opentelemetry.io/collector/config" ) -// ValueSource is an interface that helps to retrieve a config map and watch for any +// ValueProvider is an interface that helps to retrieve a config map and watch for any // changes to the config map. Implementations may load the config from a file, // a database or any other source. -type ValueSource interface { - Shutdownable +type ValueProvider interface { + BaseProvider // Retrieve goes to the configuration source and retrieves the selected data which // contains the value to be injected in the configuration and the corresponding watcher that diff --git a/config/configunmarshaler/defaultunmarshaler.go b/config/configunmarshaler/defaultunmarshaler.go index 345e95ea104..93e1e6472d7 100644 --- a/config/configunmarshaler/defaultunmarshaler.go +++ b/config/configunmarshaler/defaultunmarshaler.go @@ -71,7 +71,7 @@ type configSettings struct { Exporters map[config.ComponentID]map[string]interface{} `mapstructure:"exporters"` Extensions map[config.ComponentID]map[string]interface{} `mapstructure:"extensions"` ConfigSources map[config.ComponentID]map[string]interface{} `mapstructure:"config_sources"` - ValueSources map[config.ComponentID]map[string]interface{} `mapstructure:"value_sources"` + MergeConfigs []string `mapstructure:"merge_configs"` Service map[string]interface{} `mapstructure:"service"` } diff --git a/configsource/env/factory.go b/configsource/env/factory.go index 0ccbb7ca06b..df81a000570 100644 --- a/configsource/env/factory.go +++ b/configsource/env/factory.go @@ -23,10 +23,10 @@ func (f factory) CreateDefaultConfig() config.ConfigSource { } } -func (f factory) CreateValueSource(ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.ValueSource, error) { +func (f factory) CreateConfigSource(ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.BaseProvider, error) { return &configSource{}, nil } -func NewFactory() component.ValueSourceFactory { +func NewFactory() component.ConfigSourceFactory { return &factory{} } diff --git a/configsource/files/factory.go b/configsource/files/factory.go index 916dee38b89..18145c285ae 100644 --- a/configsource/files/factory.go +++ b/configsource/files/factory.go @@ -25,7 +25,7 @@ func (f factory) CreateDefaultConfig() config.ConfigSource { func (f factory) CreateConfigSource( ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource, -) (configmapprovider.ConfigSource, error) { +) (configmapprovider.BaseProvider, error) { return configmapprovider.NewFile(cfg.(*Config).Name), nil } diff --git a/service/config_watcher.go b/service/config_watcher.go index 232b03bedd7..58d099f8ff3 100644 --- a/service/config_watcher.go +++ b/service/config_watcher.go @@ -42,18 +42,6 @@ func newConfigWatcher(ctx context.Context, set CollectorSettings) (*configWatche return nil, fmt.Errorf("cannot get the configuration: %w", err) } - //sources, err := set.ConfigUnmarshaler.UnmarshalSources(m, set.Factories) - //if err != nil { - // return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) - //} - // - //cfgMap := config.NewMap() - //err := retrieveFromSources(ctx, cfgMap, sources, cm.onChange) - //if err != nil { - // return nil, fmt.Errorf("cannot retrieve the configuration: %w", err) - //} - //err := mergeConfigMap(cfgMap, m) - var cfg *config.Config if cfg, err = set.ConfigUnmarshaler.Unmarshal(m, set.Factories); err != nil { return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) @@ -69,15 +57,6 @@ func newConfigWatcher(ctx context.Context, set CollectorSettings) (*configWatche return cm, nil } -//func retrieveFromSources(ctx context.Context, cfgMap *config.Map, sources []component.ConfigSource, onChange func(*configmapprovider.ChangeEvent)) error { -// for _, source := range sources { -// retrieved, err := source.Retrieve(ctx, onChange) -// m, err := retrieved.Get(ctx) -// err := mergeConfigMap(cfgMap, m) -// } -// return nil -//} - func (cm *configWatcher) onChange(event *configmapprovider.ChangeEvent) { if event.Error != configsource.ErrSessionClosed { cm.watcher <- event.Error diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index abd1c91e822..705371de02f 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -26,12 +26,12 @@ import ( ) type defaultConfigProvider struct { - initial configmapprovider.ConfigSource - merged configmapprovider.ConfigSource + initial configmapprovider.Provider + merged configmapprovider.Provider factories component.Factories } -func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.ConfigSource { +func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.Provider { localProvider := configmapprovider.NewLocal(configFileName, properties) return &defaultConfigProvider{initial: localProvider, factories: factories} } @@ -43,33 +43,59 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve } rootMap, err := r.Get(ctx) - configSources, valueSources, err := unmarshalSources(ctx, rootMap, mp.factories) + configSources, mergeConfigs, err := unmarshalSources(ctx, rootMap, mp.factories) if err != nil { return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) } - configSources = append(configSources, configmapprovider.NewSimple(rootMap)) - mp.merged = configmapprovider.NewMerge(configSources...) + var rootProviders []configmapprovider.Provider + for _, configSource := range mergeConfigs { + configSourceID, err := config.NewComponentIDFromString(configSource) + if err != nil { + return nil, err + } + + configSource, ok := configSources[configSourceID] + if !ok { + return nil, fmt.Errorf("config source %q must be defined in config_sources section", configSourceID) + } + + mapProvider, ok := configSource.(configmapprovider.Provider) + if !ok { + return nil, fmt.Errorf("config source %q cannot be used in merge_configs section since it does not implement Provider interface", configSourceID) + } + + rootProviders = append(rootProviders, mapProvider) + } + + rootProviders = append(rootProviders, configmapprovider.NewSimple(rootMap)) + mp.merged = configmapprovider.NewMerge(rootProviders...) retrieved, err := mp.merged.Retrieve(ctx, onChange) if err != nil { return nil, fmt.Errorf("cannot retrive the configuration: %w", err) } + // Get list of value sources. + valueSources := map[config.ComponentID]configmapprovider.ValueProvider{} + for configSourceID, configSource := range configSources { + valueSource, ok := configSource.(configmapprovider.ValueProvider) + if ok { + valueSources[configSourceID] = valueSource + } + } + return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved, valueSources: valueSources}, nil } func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( - configSources []configmapprovider.ConfigSource, - valueSources map[config.ComponentID]configmapprovider.ValueSource, + configSources map[config.ComponentID]configmapprovider.BaseProvider, + mergeConfigs []string, err error, ) { - // Unmarshal the "config_sources" section of rootMap and create a Shutdownable for each - // config source using the factories.ConfigSources. - type RootConfig struct { ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` - ValueSources map[string]map[string]interface{} `mapstructure:"value_sources"` + MergeConfigs []string `mapstructure:"merge_configs"` } var rootCfg RootConfig err = rootMap.Unmarshal(&rootCfg) @@ -77,15 +103,24 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon return nil, nil, err } + // Unmarshal the "config_sources" section of rootMap and create a BaseProvider for each + // config source using the factories.ConfigSources. + configSources = map[config.ComponentID]configmapprovider.BaseProvider{} for _, sourceCfg := range rootCfg.ConfigSources { - for sourceType, settings := range sourceCfg { - factoryBase, ok := factories.ConfigSources[config.Type(sourceType)] + for sourceName, settings := range sourceCfg { + id, err := config.NewComponentIDFromString(sourceName) + if err != nil { + return nil, nil, err + } + sourceType := id.Type() + + factoryBase, ok := factories.ConfigSources[sourceType] if !ok { - return nil, nil, fmt.Errorf("unknown source type %q", sourceType) + return nil, nil, fmt.Errorf("unknown config source type %q (did you register the config source factory?)", sourceType) } cfg := factoryBase.CreateDefaultConfig() - cfg.SetIDName(sourceType) + cfg.SetIDName(sourceName) // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. @@ -102,44 +137,11 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon if err != nil { return nil, nil, err } - configSources = append(configSources, source) - } - } - - valueSources = map[config.ComponentID]configmapprovider.ValueSource{} - for sourceName, settings := range rootCfg.ValueSources { - id, err := config.NewComponentIDFromString(sourceName) - if err != nil { - return nil, nil, err - } - sourceType := id.Type() - factoryBase, ok := factories.ConfigSources[sourceType] - if !ok { - return nil, nil, fmt.Errorf("unknown source type %q", sourceType) - } - - cfg := factoryBase.CreateDefaultConfig() - cfg.SetIDName(id.Name()) - - // Now that the default config struct is created we can Unmarshal into it, - // and it will apply user-defined config on top of the default. - if err := configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { - return nil, nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) - } - - factory, ok := factoryBase.(component.ValueSourceFactory) - if !ok { - return nil, nil, fmt.Errorf("config source %q does not implement ValueSourceFactory", sourceType) - } - - source, err := factory.CreateValueSource(ctx, component.ConfigSourceCreateSettings{}, cfg) - if err != nil { - return nil, nil, err + configSources[id] = source } - valueSources[cfg.ID()] = source } - return configSources, valueSources, nil + return configSources, rootCfg.MergeConfigs, nil } func (mp *defaultConfigProvider) Shutdown(ctx context.Context) error { diff --git a/service/defaultconfigprovider/valuesourcesubstitutor.go b/service/defaultconfigprovider/valuesourcesubstitutor.go index 1ae80d6987b..1ff82adf4b3 100644 --- a/service/defaultconfigprovider/valuesourcesubstitutor.go +++ b/service/defaultconfigprovider/valuesourcesubstitutor.go @@ -34,7 +34,7 @@ type ( type valueSourceSubstitutor struct { onChange func(event *configmapprovider.ChangeEvent) retrieved configmapprovider.RetrievedConfig - valueSources map[config.ComponentID]configmapprovider.ValueSource + valueSources map[config.ComponentID]configmapprovider.ValueProvider } func (vp *valueSourceSubstitutor) Get(ctx context.Context) (*config.Map, error) { diff --git a/service/settings.go b/service/settings.go index c945a720107..59a16de4fc5 100644 --- a/service/settings.go +++ b/service/settings.go @@ -64,7 +64,7 @@ type CollectorSettings struct { // from a config file define by the --config command line flag and overrides component's configuration // properties supplied via --set command line flag. // If the provider is configmapprovider.WatchableRetrieved, collector may reload the configuration upon error. - ConfigMapProvider configmapprovider.ConfigSource + ConfigMapProvider configmapprovider.Provider // ConfigUnmarshaler unmarshalls the configuration's Parser into the service configuration. // If it is not provided a default unmarshaler is used. From 6363a1c4f2d009f130e66ddde399cbd3a6128843 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:27:21 -0500 Subject: [PATCH 05/23] Improve error message when config source is not ValueProvider --- .../defaultconfigprovider/defaultconfigprovider.go | 11 +---------- .../defaultconfigprovider/valuesourcesubstitutor.go | 13 +++++++++---- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 705371de02f..04aa6b73973 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -76,16 +76,7 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve return nil, fmt.Errorf("cannot retrive the configuration: %w", err) } - // Get list of value sources. - valueSources := map[config.ComponentID]configmapprovider.ValueProvider{} - for configSourceID, configSource := range configSources { - valueSource, ok := configSource.(configmapprovider.ValueProvider) - if ok { - valueSources[configSourceID] = valueSource - } - } - - return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved, valueSources: valueSources}, nil + return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved, configSources: configSources}, nil } func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( diff --git a/service/defaultconfigprovider/valuesourcesubstitutor.go b/service/defaultconfigprovider/valuesourcesubstitutor.go index 1ff82adf4b3..30e748dd2ba 100644 --- a/service/defaultconfigprovider/valuesourcesubstitutor.go +++ b/service/defaultconfigprovider/valuesourcesubstitutor.go @@ -32,9 +32,9 @@ type ( ) type valueSourceSubstitutor struct { - onChange func(event *configmapprovider.ChangeEvent) - retrieved configmapprovider.RetrievedConfig - valueSources map[config.ComponentID]configmapprovider.ValueProvider + onChange func(event *configmapprovider.ChangeEvent) + retrieved configmapprovider.RetrievedConfig + configSources map[config.ComponentID]configmapprovider.BaseProvider } func (vp *valueSourceSubstitutor) Get(ctx context.Context) (*config.Map, error) { @@ -245,11 +245,16 @@ func (vp *valueSourceSubstitutor) retrieveConfigSourceData(ctx context.Context, return nil, err } - valueSrc, ok := vp.valueSources[cfgSrcID] + configSrc, ok := vp.configSources[cfgSrcID] if !ok { return nil, newErrUnknownConfigSource(cfgSrcName) } + valueSrc, ok := configSrc.(configmapprovider.ValueProvider) + if !ok { + return nil, fmt.Errorf("config source %q is not a ValueProvider, cannot use with ${%v:...} syntax", cfgSrcName, cfgSrcName) + } + cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(cfgSrcInvocation) if err != nil { return nil, err From 8e2b97999f80a2c79735ab4669fc061909a345b6 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:34:07 -0500 Subject: [PATCH 06/23] Fix tests --- config/configmapprovider/baseprovider.go | 4 ++-- config/configmapprovider/default.go | 2 +- config/configmapprovider/expand.go | 4 ++-- config/configmapprovider/file.go | 2 +- config/configmapprovider/inmemory.go | 4 ++-- config/configmapprovider/merge_test.go | 2 +- config/configmapprovider/properties.go | 2 +- config/configmapprovider/valueprovider.go | 2 +- service/config_watcher_test.go | 12 ++++++------ 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/config/configmapprovider/baseprovider.go b/config/configmapprovider/baseprovider.go index fcb2782ffc7..0144ded6cf9 100644 --- a/config/configmapprovider/baseprovider.go +++ b/config/configmapprovider/baseprovider.go @@ -22,8 +22,8 @@ import ( // changes to the config map. Implementations may load the config from a file, // a database or any other source. type BaseProvider interface { - // Shutdown signals that the configuration for which this Shutdownable was used to - // retrieve values is no longer in use and the Shutdownable should close and release + // Shutdown signals that the configuration for which this BaseProvider was used to + // retrieve values is no longer in use and the BaseProvider should close and release // any resources that it may have created. // // This method must be called when the Collector service ends, either in case of diff --git a/config/configmapprovider/default.go b/config/configmapprovider/default.go index 1ae8f2288fd..9f1c1a2d971 100644 --- a/config/configmapprovider/default.go +++ b/config/configmapprovider/default.go @@ -14,7 +14,7 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" -// NewLocal returns the default Shutdownable, and it creates configuration from a file +// NewLocal returns the default Provider, and it creates configuration from a file // defined by the given configFile and overwrites fields using properties. func NewLocal(configFileName string, properties []string) Provider { return NewMerge( diff --git a/config/configmapprovider/expand.go b/config/configmapprovider/expand.go index 0824b4d87e4..5bb90b077e1 100644 --- a/config/configmapprovider/expand.go +++ b/config/configmapprovider/expand.go @@ -25,8 +25,8 @@ type expandMapProvider struct { base Provider } -// NewExpand returns a Shutdownable, that expands all environment variables for a -// config.Map provided by the given Shutdownable. +// NewExpand returns a Provider, that expands all environment variables for a +// config.Map provided by the given Provider. func NewExpand(base Provider) Provider { return &expandMapProvider{ base: base, diff --git a/config/configmapprovider/file.go b/config/configmapprovider/file.go index efbd5efeaa8..cc6bff9c676 100644 --- a/config/configmapprovider/file.go +++ b/config/configmapprovider/file.go @@ -26,7 +26,7 @@ type fileMapProvider struct { fileName string } -// NewFile returns a new Shutdownable that reads the configuration from the given file. +// NewFile returns a new Provider that reads the configuration from the given file. func NewFile(fileName string) Provider { return &fileMapProvider{ fileName: fileName, diff --git a/config/configmapprovider/inmemory.go b/config/configmapprovider/inmemory.go index 43d6ab9f27d..07d1db52276 100644 --- a/config/configmapprovider/inmemory.go +++ b/config/configmapprovider/inmemory.go @@ -25,8 +25,8 @@ type inMemoryMapProvider struct { buf io.Reader } -// NewInMemory returns a new Shutdownable that reads the configuration, from the provided buffer, as YAML. -func NewInMemory(buf io.Reader) BaseProvider { +// NewInMemory returns a new BaseProvider that reads the configuration, from the provided buffer, as YAML. +func NewInMemory(buf io.Reader) Provider { return &inMemoryMapProvider{buf: buf} } diff --git a/config/configmapprovider/merge_test.go b/config/configmapprovider/merge_test.go index c556b4e00cc..415c036ad7e 100644 --- a/config/configmapprovider/merge_test.go +++ b/config/configmapprovider/merge_test.go @@ -43,7 +43,7 @@ type errProvider struct { err error } -func (epl *errProvider) Retrieve(context.Context, func(*ChangeEvent)) (Retrieved, error) { +func (epl *errProvider) Retrieve(context.Context, func(*ChangeEvent)) (RetrievedConfig, error) { if epl.err == nil { return &simpleRetrieved{confMap: config.NewMap()}, nil } diff --git a/config/configmapprovider/properties.go b/config/configmapprovider/properties.go index 1c0fea0c347..172fc21ac1f 100644 --- a/config/configmapprovider/properties.go +++ b/config/configmapprovider/properties.go @@ -29,7 +29,7 @@ type propertiesMapProvider struct { properties []string } -// NewProperties returns a Shutdownable, that provides a config.Map from the given properties. +// NewProperties returns a Provider, that provides a config.Map from the given properties. // // Properties must follow the Java properties format, key-value list separated by equal sign with a "." // as key delimiter. diff --git a/config/configmapprovider/valueprovider.go b/config/configmapprovider/valueprovider.go index 027596f70a2..5ae1b51203e 100644 --- a/config/configmapprovider/valueprovider.go +++ b/config/configmapprovider/valueprovider.go @@ -41,7 +41,7 @@ type ValueProvider interface { Retrieve(ctx context.Context, onChange func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) } -// RetrievedValue holds the result of a call to the Retrieve method of a Shutdownable object. +// RetrievedValue holds the result of a call to the Retrieve method of a ValueProvider object. // // The typical usage is the following: // diff --git a/service/config_watcher_test.go b/service/config_watcher_test.go index b68ae6d3b0d..e58bbf94b0a 100644 --- a/service/config_watcher_test.go +++ b/service/config_watcher_test.go @@ -37,7 +37,7 @@ type errConfigMapProvider struct { err error } -func (ecmp *errConfigMapProvider) Retrieve(_ context.Context, onChange func(*configmapprovider.ChangeEvent)) (configmapprovider.Retrieved, error) { +func (ecmp *errConfigMapProvider) Retrieve(_ context.Context, onChange func(*configmapprovider.ChangeEvent)) (configmapprovider.RetrievedConfig, error) { if ecmp.ret != nil { ecmp.ret.onChange = onChange } @@ -78,7 +78,7 @@ func TestConfigWatcher(t *testing.T) { tests := []struct { name string - parserProvider configmapprovider.Shutdownable + parserProvider configmapprovider.Provider configUnmarshaler configunmarshaler.ConfigUnmarshaler expectNewErr bool expectWatchErr bool @@ -104,7 +104,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "watch_err", - parserProvider: func() configmapprovider.Shutdownable { + parserProvider: func() configmapprovider.Provider { ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) m, err := ret.Get(context.Background()) @@ -116,7 +116,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "close_err", - parserProvider: func() configmapprovider.Shutdownable { + parserProvider: func() configmapprovider.Provider { ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) m, err := ret.Get(context.Background()) @@ -128,7 +128,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "ok", - parserProvider: func() configmapprovider.Shutdownable { + parserProvider: func() configmapprovider.Provider { // Use errRetrieved with nil errors to have Watchable interface implemented. ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestConfigWatcherWhenClosed(t *testing.T) { factories, errF := componenttest.NopFactories() require.NoError(t, errF) set := CollectorSettings{ - ConfigMapProvider: func() configmapprovider.Shutdownable { + ConfigMapProvider: func() configmapprovider.Provider { // Use errRetrieved with nil errors to have Watchable interface implemented. ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) From 30b13ddd5d2cad7efbe69ba4160a242327a038e5 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:35:28 -0500 Subject: [PATCH 07/23] Rename to valueSubstitutor --- .../defaultconfigprovider/defaultconfigprovider.go | 2 +- ...luesourcesubstitutor.go => valuesubstitutor.go} | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) rename service/defaultconfigprovider/{valuesourcesubstitutor.go => valuesubstitutor.go} (95%) diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 04aa6b73973..fec02f42859 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -76,7 +76,7 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve return nil, fmt.Errorf("cannot retrive the configuration: %w", err) } - return &valueSourceSubstitutor{onChange: onChange, retrieved: retrieved, configSources: configSources}, nil + return &valueSubstitutor{onChange: onChange, retrieved: retrieved, configSources: configSources}, nil } func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( diff --git a/service/defaultconfigprovider/valuesourcesubstitutor.go b/service/defaultconfigprovider/valuesubstitutor.go similarity index 95% rename from service/defaultconfigprovider/valuesourcesubstitutor.go rename to service/defaultconfigprovider/valuesubstitutor.go index 30e748dd2ba..f31cb0d747e 100644 --- a/service/defaultconfigprovider/valuesourcesubstitutor.go +++ b/service/defaultconfigprovider/valuesubstitutor.go @@ -31,13 +31,13 @@ type ( errUnknownConfigSource struct{ error } ) -type valueSourceSubstitutor struct { +type valueSubstitutor struct { onChange func(event *configmapprovider.ChangeEvent) retrieved configmapprovider.RetrievedConfig configSources map[config.ComponentID]configmapprovider.BaseProvider } -func (vp *valueSourceSubstitutor) Get(ctx context.Context) (*config.Map, error) { +func (vp *valueSubstitutor) Get(ctx context.Context) (*config.Map, error) { cfgMap, err := vp.retrieved.Get(ctx) if err != nil { return nil, err @@ -46,7 +46,7 @@ func (vp *valueSourceSubstitutor) Get(ctx context.Context) (*config.Map, error) return vp.substitute(ctx, cfgMap) } -func (vp *valueSourceSubstitutor) substitute(ctx context.Context, cfgMap *config.Map) (*config.Map, error) { +func (vp *valueSubstitutor) substitute(ctx context.Context, cfgMap *config.Map) (*config.Map, error) { for _, k := range cfgMap.AllKeys() { val, err := vp.parseConfigValue(ctx, cfgMap.Get(k)) if err != nil { @@ -58,14 +58,14 @@ func (vp *valueSourceSubstitutor) substitute(ctx context.Context, cfgMap *config return cfgMap, nil } -func (vp *valueSourceSubstitutor) Close(ctx context.Context) error { +func (vp *valueSubstitutor) Close(ctx context.Context) error { return vp.retrieved.Close(ctx) } // parseConfigValue takes the value of a "config node" and process it recursively. The processing consists // in transforming invocations of config sources and/or environment variables into literal data that can be // used directly from a `config.Map` object. -func (vp *valueSourceSubstitutor) parseConfigValue(ctx context.Context, value interface{}) (interface{}, error) { +func (vp *valueSubstitutor) parseConfigValue(ctx context.Context, value interface{}) (interface{}, error) { switch v := value.(type) { case string: // Only if the value of the node is a string it can contain an env var or config source @@ -114,7 +114,7 @@ func (vp *valueSourceSubstitutor) parseConfigValue(ctx context.Context, value in // parseStringValue transforms environment variables and config sources, if any are present, on // the given string in the configuration into an object to be inserted into the resulting configuration. -func (vp *valueSourceSubstitutor) parseStringValue(ctx context.Context, s string) (interface{}, error) { +func (vp *valueSubstitutor) parseStringValue(ctx context.Context, s string) (interface{}, error) { // Code based on os.Expand function. All delimiters that are checked against are // ASCII so bytes are fine for this operation. var buf []byte @@ -239,7 +239,7 @@ func (vp *valueSourceSubstitutor) parseStringValue(ctx context.Context, s string // retrieveConfigSourceData retrieves data from the specified config source and injects them into // the configuration. The Manager tracks sessions and watcher objects as needed. -func (vp *valueSourceSubstitutor) retrieveConfigSourceData(ctx context.Context, cfgSrcName string, cfgSrcInvocation string) (interface{}, error) { +func (vp *valueSubstitutor) retrieveConfigSourceData(ctx context.Context, cfgSrcName string, cfgSrcInvocation string) (interface{}, error) { cfgSrcID, err := config.NewComponentIDFromString(cfgSrcName) if err != nil { return nil, err From 9f9109231a0d2f3d5467f6d3893c590bb8640ad0 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:41:16 -0500 Subject: [PATCH 08/23] Improve interface names --- component/configsource.go | 4 +- config/configmapprovider/baseprovider.go | 44 ---------- config/configmapprovider/default.go | 4 +- config/configmapprovider/expand.go | 10 +-- config/configmapprovider/file.go | 6 +- config/configmapprovider/inmemory.go | 6 +- config/configmapprovider/mapprovider.go | 79 ++++++++++++++++++ config/configmapprovider/merge.go | 8 +- config/configmapprovider/merge_test.go | 2 +- config/configmapprovider/properties.go | 6 +- config/configmapprovider/provider.go | 81 +++++++------------ config/configmapprovider/simple.go | 4 +- config/configmapprovider/valueprovider.go | 10 +-- configsource/env/factory.go | 2 +- configsource/files/factory.go | 2 +- service/config_watcher.go | 2 +- service/config_watcher_test.go | 12 +-- .../defaultconfigprovider.go | 20 ++--- .../defaultconfigprovider/valuesubstitutor.go | 4 +- service/settings.go | 2 +- 20 files changed, 161 insertions(+), 147 deletions(-) delete mode 100644 config/configmapprovider/baseprovider.go create mode 100644 config/configmapprovider/mapprovider.go diff --git a/component/configsource.go b/component/configsource.go index a2a4520ae42..c19eee5dca1 100644 --- a/component/configsource.go +++ b/component/configsource.go @@ -23,7 +23,7 @@ import ( // ConfigSource is type ConfigSource interface { - configmapprovider.BaseProvider + configmapprovider.Provider } // ConfigSourceCreateSettings is passed to ExtensionFactory.Create* functions. @@ -35,5 +35,5 @@ type ConfigSourceCreateSettings struct { type ConfigSourceFactory interface { Factory CreateDefaultConfig() config.ConfigSource - CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.BaseProvider, error) + CreateConfigSource(ctx context.Context, set ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.Provider, error) } diff --git a/config/configmapprovider/baseprovider.go b/config/configmapprovider/baseprovider.go deleted file mode 100644 index 0144ded6cf9..00000000000 --- a/config/configmapprovider/baseprovider.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" - -import ( - "context" -) - -// BaseProvider is an interface that helps to retrieve a config map and watch for any -// changes to the config map. Implementations may load the config from a file, -// a database or any other source. -type BaseProvider interface { - // Shutdown signals that the configuration for which this BaseProvider was used to - // retrieve values is no longer in use and the BaseProvider should close and release - // any resources that it may have created. - // - // This method must be called when the Collector service ends, either in case of - // success or error. Retrieve cannot be called after Shutdown. - // - // Should never be called concurrently with itself or with Retrieve. - // If ctx is cancelled should return immediately with an error. - Shutdown(ctx context.Context) error -} - -// ChangeEvent describes the particular change event that happened with the config. -// TODO: see if this can be eliminated. -type ChangeEvent struct { - // Error is nil if the config is changed and needs to be re-fetched using Get. - // It is set to configsource.ErrSessionClosed if Close was called. - // Any other error indicates that there was a problem with retrieving the config. - Error error -} diff --git a/config/configmapprovider/default.go b/config/configmapprovider/default.go index 9f1c1a2d971..c4e871f935b 100644 --- a/config/configmapprovider/default.go +++ b/config/configmapprovider/default.go @@ -14,9 +14,9 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" -// NewLocal returns the default Provider, and it creates configuration from a file +// NewLocal returns the default MapProvider, and it creates configuration from a file // defined by the given configFile and overwrites fields using properties. -func NewLocal(configFileName string, properties []string) Provider { +func NewLocal(configFileName string, properties []string) MapProvider { return NewMerge( NewFile(configFileName), NewProperties(properties)) diff --git a/config/configmapprovider/expand.go b/config/configmapprovider/expand.go index 5bb90b077e1..239ecd3ad93 100644 --- a/config/configmapprovider/expand.go +++ b/config/configmapprovider/expand.go @@ -22,18 +22,18 @@ import ( ) type expandMapProvider struct { - base Provider + base MapProvider } -// NewExpand returns a Provider, that expands all environment variables for a -// config.Map provided by the given Provider. -func NewExpand(base Provider) Provider { +// NewExpand returns a MapProvider, that expands all environment variables for a +// config.Map provided by the given MapProvider. +func NewExpand(base MapProvider) MapProvider { return &expandMapProvider{ base: base, } } -func (emp *expandMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { +func (emp *expandMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { retr, err := emp.base.Retrieve(ctx, onChange) if err != nil { return nil, fmt.Errorf("failed to retrieve from base provider: %w", err) diff --git a/config/configmapprovider/file.go b/config/configmapprovider/file.go index cc6bff9c676..88cc7b529cc 100644 --- a/config/configmapprovider/file.go +++ b/config/configmapprovider/file.go @@ -26,14 +26,14 @@ type fileMapProvider struct { fileName string } -// NewFile returns a new Provider that reads the configuration from the given file. -func NewFile(fileName string) Provider { +// NewFile returns a new MapProvider that reads the configuration from the given file. +func NewFile(fileName string) MapProvider { return &fileMapProvider{ fileName: fileName, } } -func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (RetrievedConfig, error) { +func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (RetrievedMap, error) { if fmp.fileName == "" { return nil, errors.New("config file not specified") } diff --git a/config/configmapprovider/inmemory.go b/config/configmapprovider/inmemory.go index 07d1db52276..7a5ebd4db28 100644 --- a/config/configmapprovider/inmemory.go +++ b/config/configmapprovider/inmemory.go @@ -25,12 +25,12 @@ type inMemoryMapProvider struct { buf io.Reader } -// NewInMemory returns a new BaseProvider that reads the configuration, from the provided buffer, as YAML. -func NewInMemory(buf io.Reader) Provider { +// NewInMemory returns a new Provider that reads the configuration, from the provided buffer, as YAML. +func NewInMemory(buf io.Reader) MapProvider { return &inMemoryMapProvider{buf: buf} } -func (inp *inMemoryMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { +func (inp *inMemoryMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { cfg, err := config.NewMapFromBuffer(inp.buf) if err != nil { return nil, err diff --git a/config/configmapprovider/mapprovider.go b/config/configmapprovider/mapprovider.go new file mode 100644 index 00000000000..4e4d4058790 --- /dev/null +++ b/config/configmapprovider/mapprovider.go @@ -0,0 +1,79 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmapprovider + +import ( + "context" + + "go.opentelemetry.io/collector/config" +) + +// MapProvider is an interface that helps to retrieve a config map and watch for any +// changes to the config map. Implementations may load the config from a file, +// a database or any other source. +type MapProvider interface { + Provider + + // Retrieve goes to the configuration source and retrieves the selected data which + // contains the value to be injected in the configuration and the corresponding watcher that + // will be used to monitor for updates of the retrieved value. + // + // onChange callback is called when the config changes. onChange may be called from + // a different go routine. After onChange is called Retrieved.Get should be called + // to get the new config. See description of Retrieved for more details. + // onChange may be nil, which indicates that the caller is not interested in + // knowing about the changes. + // + // If ctx is cancelled should return immediately with an error. + // Should never be called concurrently with itself or with Shutdown. + Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) +} + +// RetrievedMap holds the result of a call to the Retrieve method of a Provider object. +// +// The typical usage is the following: +// +// r := mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// r = mapProvider.Retrieve() +// r.Get() +// // wait for onChange() to be called. +// r.Close() +// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. +// // ... +// mapProvider.Shutdown() +type RetrievedMap interface { + // Get returns the config Map. + // If Close is called before Get or concurrently with Get then Get + // should return immediately with ErrSessionClosed error. + // Should never be called concurrently with itself. + // If ctx is cancelled should return immediately with an error. + Get(ctx context.Context) (*config.Map, error) + + // Close signals that the configuration for which it was used to retrieve values is + // no longer in use and the object should close and release any watchers that it + // may have created. + // + // This method must be called when the service ends, either in case of success or error. + // + // Should never be called concurrently with itself. + // May be called before, after or concurrently with Get. + // If ctx is cancelled should return immediately with an error. + // + // Calling Close on an already closed object should have no effect and should return nil. + Close(ctx context.Context) error +} diff --git a/config/configmapprovider/merge.go b/config/configmapprovider/merge.go index ff68fed2ede..42b4e1675c7 100644 --- a/config/configmapprovider/merge.go +++ b/config/configmapprovider/merge.go @@ -25,17 +25,17 @@ import ( // TODO: Add support to "merge" watchable interface. type mergeMapProvider struct { - providers []Provider + providers []MapProvider } -// NewMerge returns a Provider, that merges the result from multiple Provider(s). +// NewMerge returns a MapProvider, that merges the result from multiple MapProvider(s). // // The ConfigMaps are merged in the given order, by merging all of them in order into an initial empty map. -func NewMerge(ps ...Provider) Provider { +func NewMerge(ps ...MapProvider) MapProvider { return &mergeMapProvider{providers: ps} } -func (mp *mergeMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { +func (mp *mergeMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { retCfgMap := config.NewMap() for _, p := range mp.providers { retr, err := p.Retrieve(ctx, onChange) diff --git a/config/configmapprovider/merge_test.go b/config/configmapprovider/merge_test.go index 415c036ad7e..7f486f23762 100644 --- a/config/configmapprovider/merge_test.go +++ b/config/configmapprovider/merge_test.go @@ -43,7 +43,7 @@ type errProvider struct { err error } -func (epl *errProvider) Retrieve(context.Context, func(*ChangeEvent)) (RetrievedConfig, error) { +func (epl *errProvider) Retrieve(context.Context, func(*ChangeEvent)) (RetrievedMap, error) { if epl.err == nil { return &simpleRetrieved{confMap: config.NewMap()}, nil } diff --git a/config/configmapprovider/properties.go b/config/configmapprovider/properties.go index 172fc21ac1f..3829f153b94 100644 --- a/config/configmapprovider/properties.go +++ b/config/configmapprovider/properties.go @@ -29,18 +29,18 @@ type propertiesMapProvider struct { properties []string } -// NewProperties returns a Provider, that provides a config.Map from the given properties. +// NewProperties returns a MapProvider, that provides a config.Map from the given properties. // // Properties must follow the Java properties format, key-value list separated by equal sign with a "." // as key delimiter. // ["processors.batch.timeout=2s", "processors.batch/foo.timeout=3s"] -func NewProperties(properties []string) Provider { +func NewProperties(properties []string) MapProvider { return &propertiesMapProvider{ properties: properties, } } -func (pmp *propertiesMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { +func (pmp *propertiesMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { if len(pmp.properties) == 0 { return &simpleRetrieved{confMap: config.NewMap()}, nil } diff --git a/config/configmapprovider/provider.go b/config/configmapprovider/provider.go index f3aafc34ba7..271bd065c0e 100644 --- a/config/configmapprovider/provider.go +++ b/config/configmapprovider/provider.go @@ -1,65 +1,44 @@ -package configmapprovider +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" import ( "context" - - "go.opentelemetry.io/collector/config" ) -// Provider is an interface that helps to retrieve a config map and watch for any +// Provider is an interface that helps to retrieve a config map or value and watch for any // changes to the config map. Implementations may load the config from a file, // a database or any other source. type Provider interface { - BaseProvider - - // Retrieve goes to the configuration source and retrieves the selected data which - // contains the value to be injected in the configuration and the corresponding watcher that - // will be used to monitor for updates of the retrieved value. + // Shutdown signals that the configuration for which this Provider was used to + // retrieve values is no longer in use and the Provider should close and release + // any resources that it may have created. // - // onChange callback is called when the config changes. onChange may be called from - // a different go routine. After onChange is called Retrieved.Get should be called - // to get the new config. See description of Retrieved for more details. - // onChange may be nil, which indicates that the caller is not interested in - // knowing about the changes. + // This method must be called when the Collector service ends, either in case of + // success or error. Retrieve cannot be called after Shutdown. // + // Should never be called concurrently with itself or with Retrieve. // If ctx is cancelled should return immediately with an error. - // Should never be called concurrently with itself or with Shutdown. - Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) + Shutdown(ctx context.Context) error } -// RetrievedConfig holds the result of a call to the Retrieve method of a BaseProvider object. -// -// The typical usage is the following: -// -// r := mapProvider.Retrieve() -// r.Get() -// // wait for onChange() to be called. -// r.Close() -// r = mapProvider.Retrieve() -// r.Get() -// // wait for onChange() to be called. -// r.Close() -// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. -// // ... -// mapProvider.Shutdown() -type RetrievedConfig interface { - // Get returns the config Map. - // If Close is called before Get or concurrently with Get then Get - // should return immediately with ErrSessionClosed error. - // Should never be called concurrently with itself. - // If ctx is cancelled should return immediately with an error. - Get(ctx context.Context) (*config.Map, error) - - // Close signals that the configuration for which it was used to retrieve values is - // no longer in use and the object should close and release any watchers that it - // may have created. - // - // This method must be called when the service ends, either in case of success or error. - // - // Should never be called concurrently with itself. - // May be called before, after or concurrently with Get. - // If ctx is cancelled should return immediately with an error. - // - // Calling Close on an already closed object should have no effect and should return nil. - Close(ctx context.Context) error +// ChangeEvent describes the particular change event that happened with the config. +// TODO: see if this can be eliminated. +type ChangeEvent struct { + // Error is nil if the config is changed and needs to be re-fetched using Get. + // It is set to configsource.ErrSessionClosed if Close was called. + // Any other error indicates that there was a problem with retrieving the config. + Error error } diff --git a/config/configmapprovider/simple.go b/config/configmapprovider/simple.go index 7fb4fa2192e..4ec62ec8f98 100644 --- a/config/configmapprovider/simple.go +++ b/config/configmapprovider/simple.go @@ -25,7 +25,7 @@ type simpleProvider struct { confMap *config.Map } -func (s simpleProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedConfig, error) { +func (s simpleProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { return &simpleRetrieved{confMap: s.confMap}, nil } @@ -33,7 +33,7 @@ func (s simpleProvider) Shutdown(ctx context.Context) error { return nil } -func NewSimple(confMap *config.Map) Provider { +func NewSimple(confMap *config.Map) MapProvider { return &simpleProvider{confMap: confMap} } diff --git a/config/configmapprovider/valueprovider.go b/config/configmapprovider/valueprovider.go index 5ae1b51203e..93b410e8ca6 100644 --- a/config/configmapprovider/valueprovider.go +++ b/config/configmapprovider/valueprovider.go @@ -20,11 +20,11 @@ import ( "go.opentelemetry.io/collector/config" ) -// ValueProvider is an interface that helps to retrieve a config map and watch for any -// changes to the config map. Implementations may load the config from a file, +// ValueProvider is an interface that helps to retrieve a config value and watch for any +// changes to the config value. Implementations may load the config from a file, // a database or any other source. type ValueProvider interface { - BaseProvider + Provider // Retrieve goes to the configuration source and retrieves the selected data which // contains the value to be injected in the configuration and the corresponding watcher that @@ -45,11 +45,11 @@ type ValueProvider interface { // // The typical usage is the following: // -// r := mapProvider.Retrieve() +// r := valueProvider.Retrieve(selector, params) // r.Get() // // wait for onChange() to be called. // r.Close() -// r = mapProvider.Retrieve() +// r = valueProvider.Retrieve(selector, params) // r.Get() // // wait for onChange() to be called. // r.Close() diff --git a/configsource/env/factory.go b/configsource/env/factory.go index df81a000570..4dca6ca9b11 100644 --- a/configsource/env/factory.go +++ b/configsource/env/factory.go @@ -23,7 +23,7 @@ func (f factory) CreateDefaultConfig() config.ConfigSource { } } -func (f factory) CreateConfigSource(ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.BaseProvider, error) { +func (f factory) CreateConfigSource(ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.Provider, error) { return &configSource{}, nil } diff --git a/configsource/files/factory.go b/configsource/files/factory.go index 18145c285ae..34722c44af3 100644 --- a/configsource/files/factory.go +++ b/configsource/files/factory.go @@ -25,7 +25,7 @@ func (f factory) CreateDefaultConfig() config.ConfigSource { func (f factory) CreateConfigSource( ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource, -) (configmapprovider.BaseProvider, error) { +) (configmapprovider.Provider, error) { return configmapprovider.NewFile(cfg.(*Config).Name), nil } diff --git a/service/config_watcher.go b/service/config_watcher.go index 58d099f8ff3..541940eee47 100644 --- a/service/config_watcher.go +++ b/service/config_watcher.go @@ -25,7 +25,7 @@ import ( type configWatcher struct { cfg *config.Config - ret configmapprovider.RetrievedConfig + ret configmapprovider.RetrievedMap watcher chan error } diff --git a/service/config_watcher_test.go b/service/config_watcher_test.go index e58bbf94b0a..3ff0ed59e3e 100644 --- a/service/config_watcher_test.go +++ b/service/config_watcher_test.go @@ -37,7 +37,7 @@ type errConfigMapProvider struct { err error } -func (ecmp *errConfigMapProvider) Retrieve(_ context.Context, onChange func(*configmapprovider.ChangeEvent)) (configmapprovider.RetrievedConfig, error) { +func (ecmp *errConfigMapProvider) Retrieve(_ context.Context, onChange func(*configmapprovider.ChangeEvent)) (configmapprovider.RetrievedMap, error) { if ecmp.ret != nil { ecmp.ret.onChange = onChange } @@ -78,7 +78,7 @@ func TestConfigWatcher(t *testing.T) { tests := []struct { name string - parserProvider configmapprovider.Provider + parserProvider configmapprovider.MapProvider configUnmarshaler configunmarshaler.ConfigUnmarshaler expectNewErr bool expectWatchErr bool @@ -104,7 +104,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "watch_err", - parserProvider: func() configmapprovider.Provider { + parserProvider: func() configmapprovider.MapProvider { ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) m, err := ret.Get(context.Background()) @@ -116,7 +116,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "close_err", - parserProvider: func() configmapprovider.Provider { + parserProvider: func() configmapprovider.MapProvider { ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) m, err := ret.Get(context.Background()) @@ -128,7 +128,7 @@ func TestConfigWatcher(t *testing.T) { }, { name: "ok", - parserProvider: func() configmapprovider.Provider { + parserProvider: func() configmapprovider.MapProvider { // Use errRetrieved with nil errors to have Watchable interface implemented. ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestConfigWatcherWhenClosed(t *testing.T) { factories, errF := componenttest.NopFactories() require.NoError(t, errF) set := CollectorSettings{ - ConfigMapProvider: func() configmapprovider.Provider { + ConfigMapProvider: func() configmapprovider.MapProvider { // Use errRetrieved with nil errors to have Watchable interface implemented. ret, err := configmapprovider.NewFile(path.Join("testdata", "otelcol-nop.yaml")).Retrieve(context.Background(), nil) require.NoError(t, err) diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index fec02f42859..6c74c846965 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -26,17 +26,17 @@ import ( ) type defaultConfigProvider struct { - initial configmapprovider.Provider - merged configmapprovider.Provider + initial configmapprovider.MapProvider + merged configmapprovider.MapProvider factories component.Factories } -func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.Provider { +func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.MapProvider { localProvider := configmapprovider.NewLocal(configFileName, properties) return &defaultConfigProvider{initial: localProvider, factories: factories} } -func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(event *configmapprovider.ChangeEvent)) (configmapprovider.RetrievedConfig, error) { +func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(event *configmapprovider.ChangeEvent)) (configmapprovider.RetrievedMap, error) { r, err := mp.initial.Retrieve(ctx, onChange) if err != nil { return nil, err @@ -48,7 +48,7 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) } - var rootProviders []configmapprovider.Provider + var rootProviders []configmapprovider.MapProvider for _, configSource := range mergeConfigs { configSourceID, err := config.NewComponentIDFromString(configSource) if err != nil { @@ -60,9 +60,9 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve return nil, fmt.Errorf("config source %q must be defined in config_sources section", configSourceID) } - mapProvider, ok := configSource.(configmapprovider.Provider) + mapProvider, ok := configSource.(configmapprovider.MapProvider) if !ok { - return nil, fmt.Errorf("config source %q cannot be used in merge_configs section since it does not implement Provider interface", configSourceID) + return nil, fmt.Errorf("config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", configSourceID) } rootProviders = append(rootProviders, mapProvider) @@ -80,7 +80,7 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve } func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( - configSources map[config.ComponentID]configmapprovider.BaseProvider, + configSources map[config.ComponentID]configmapprovider.Provider, mergeConfigs []string, err error, ) { @@ -94,9 +94,9 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon return nil, nil, err } - // Unmarshal the "config_sources" section of rootMap and create a BaseProvider for each + // Unmarshal the "config_sources" section of rootMap and create a Provider for each // config source using the factories.ConfigSources. - configSources = map[config.ComponentID]configmapprovider.BaseProvider{} + configSources = map[config.ComponentID]configmapprovider.Provider{} for _, sourceCfg := range rootCfg.ConfigSources { for sourceName, settings := range sourceCfg { id, err := config.NewComponentIDFromString(sourceName) diff --git a/service/defaultconfigprovider/valuesubstitutor.go b/service/defaultconfigprovider/valuesubstitutor.go index f31cb0d747e..72b1d8d2924 100644 --- a/service/defaultconfigprovider/valuesubstitutor.go +++ b/service/defaultconfigprovider/valuesubstitutor.go @@ -33,8 +33,8 @@ type ( type valueSubstitutor struct { onChange func(event *configmapprovider.ChangeEvent) - retrieved configmapprovider.RetrievedConfig - configSources map[config.ComponentID]configmapprovider.BaseProvider + retrieved configmapprovider.RetrievedMap + configSources map[config.ComponentID]configmapprovider.Provider } func (vp *valueSubstitutor) Get(ctx context.Context) (*config.Map, error) { diff --git a/service/settings.go b/service/settings.go index 59a16de4fc5..ab93d11d352 100644 --- a/service/settings.go +++ b/service/settings.go @@ -64,7 +64,7 @@ type CollectorSettings struct { // from a config file define by the --config command line flag and overrides component's configuration // properties supplied via --set command line flag. // If the provider is configmapprovider.WatchableRetrieved, collector may reload the configuration upon error. - ConfigMapProvider configmapprovider.Provider + ConfigMapProvider configmapprovider.MapProvider // ConfigUnmarshaler unmarshalls the configuration's Parser into the service configuration. // If it is not provided a default unmarshaler is used. From b6cb36e693c1705568609efd0e4876309d375563 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:46:55 -0500 Subject: [PATCH 09/23] Delete unused fields --- component/configsource.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/component/configsource.go b/component/configsource.go index c19eee5dca1..7715deec8e7 100644 --- a/component/configsource.go +++ b/component/configsource.go @@ -26,10 +26,8 @@ type ConfigSource interface { configmapprovider.Provider } -// ConfigSourceCreateSettings is passed to ExtensionFactory.Create* functions. +// ConfigSourceCreateSettings is passed to CreateConfigSource functions. type ConfigSourceCreateSettings struct { - // BuildInfo can be used by components for informational purposes - BuildInfo BuildInfo } type ConfigSourceFactory interface { From 7959de4611d26f68fee7697b1982c86e3dfa0452 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 11:50:22 -0500 Subject: [PATCH 10/23] Rename default to local (config map provider) --- component/configsource.go | 2 +- component/factories.go | 1 + config/configmapprovider/{default.go => local.go} | 2 +- .../{default_test.go => local_test.go} | 10 +++++----- config/configsource.go | 5 +++-- 5 files changed, 11 insertions(+), 9 deletions(-) rename config/configmapprovider/{default.go => local.go} (91%) rename config/configmapprovider/{default_test.go => local_test.go} (90%) diff --git a/component/configsource.go b/component/configsource.go index 7715deec8e7..9ec69701e00 100644 --- a/component/configsource.go +++ b/component/configsource.go @@ -21,7 +21,7 @@ import ( "go.opentelemetry.io/collector/config/configmapprovider" ) -// ConfigSource is +// ConfigSource is a component that provides a configuration map or value. type ConfigSource interface { configmapprovider.Provider } diff --git a/component/factories.go b/component/factories.go index 6682dff041c..c2a9079871b 100644 --- a/component/factories.go +++ b/component/factories.go @@ -35,6 +35,7 @@ type Factories struct { // Extensions maps extension type names in the config to the respective factory. Extensions map[config.Type]ExtensionFactory + // ConfigSources maps config source type names in the config to the respective factory. ConfigSources map[config.Type]ConfigSourceFactory } diff --git a/config/configmapprovider/default.go b/config/configmapprovider/local.go similarity index 91% rename from config/configmapprovider/default.go rename to config/configmapprovider/local.go index c4e871f935b..3fe9e728b74 100644 --- a/config/configmapprovider/default.go +++ b/config/configmapprovider/local.go @@ -14,7 +14,7 @@ package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" -// NewLocal returns the default MapProvider, and it creates configuration from a file +// NewLocal returns a local MapProvider, and it creates configuration from a file // defined by the given configFile and overwrites fields using properties. func NewLocal(configFileName string, properties []string) MapProvider { return NewMerge( diff --git a/config/configmapprovider/default_test.go b/config/configmapprovider/local_test.go similarity index 90% rename from config/configmapprovider/default_test.go rename to config/configmapprovider/local_test.go index b6228d6655d..cf4130e8ee8 100644 --- a/config/configmapprovider/default_test.go +++ b/config/configmapprovider/local_test.go @@ -25,7 +25,7 @@ import ( "go.opentelemetry.io/collector/config" ) -func TestDefaultMapProvider(t *testing.T) { +func TestLocalMapProvider(t *testing.T) { mp := NewLocal("testdata/default-config.yaml", nil) retr, err := mp.Retrieve(context.Background(), nil) require.NoError(t, err) @@ -44,7 +44,7 @@ exporters: assert.NoError(t, mp.Shutdown(context.Background())) } -func TestDefaultMapProvider_AddNewConfig(t *testing.T) { +func TestLocalMapProvider_AddNewConfig(t *testing.T) { mp := NewLocal("testdata/default-config.yaml", []string{"processors.batch.timeout=2s"}) cp, err := mp.Retrieve(context.Background(), nil) require.NoError(t, err) @@ -64,7 +64,7 @@ exporters: assert.NoError(t, mp.Shutdown(context.Background())) } -func TestDefaultMapProvider_OverwriteConfig(t *testing.T) { +func TestLocalMapProvider_OverwriteConfig(t *testing.T) { mp := NewLocal( "testdata/default-config.yaml", []string{"processors.batch.timeout=2s", "exporters.otlp.endpoint=localhost:1234"}) @@ -86,7 +86,7 @@ exporters: assert.NoError(t, mp.Shutdown(context.Background())) } -func TestDefaultMapProvider_InexistentFile(t *testing.T) { +func TestLocalMapProvider_InexistentFile(t *testing.T) { mp := NewLocal("testdata/otelcol-config.yaml", nil) require.NotNil(t, mp) _, err := mp.Retrieve(context.Background(), nil) @@ -95,7 +95,7 @@ func TestDefaultMapProvider_InexistentFile(t *testing.T) { assert.NoError(t, mp.Shutdown(context.Background())) } -func TestDefaultMapProvider_EmptyFileName(t *testing.T) { +func TestLocalMapProvider_EmptyFileName(t *testing.T) { mp := NewLocal("", nil) _, err := mp.Retrieve(context.Background(), nil) require.Error(t, err) diff --git a/config/configsource.go b/config/configsource.go index 9cc8a666bc2..7e2b790cde8 100644 --- a/config/configsource.go +++ b/config/configsource.go @@ -14,8 +14,9 @@ package config // import "go.opentelemetry.io/collector/config" -// ConfigSource is the configuration of a component.ConfigSource. Specific ConfigSources must implement -// this interface and must embed ConfigSourceSettings struct or a struct that extends it. +// ConfigSource is the configuration of a component.ConfigSource. +// Specific ConfigSources must implement this interface and must embed +// ConfigSourceSettings struct or a struct that extends it. type ConfigSource interface { identifiable validatable From 521b8752618ab30e151fc9dea92a0cdd8c177a39 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 12:09:34 -0500 Subject: [PATCH 11/23] Delete unnecessary changes --- config/configmapprovider/expand.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/config/configmapprovider/expand.go b/config/configmapprovider/expand.go index 239ecd3ad93..89f460e874c 100644 --- a/config/configmapprovider/expand.go +++ b/config/configmapprovider/expand.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "os" - "strings" ) type expandMapProvider struct { @@ -80,12 +79,6 @@ func expandStringValues(value interface{}) interface{} { } func expandEnv(s string) string { - if strings.Contains(s, ":") { - // This uses the extended syntax ${valuesrc:selector} which will be expanded - // later by ValueProvider substitutor, so don't touch it. - return s - } - return os.Expand(s, func(str string) string { // This allows escaping environment variable substitution via $$, e.g. // - $FOO will be substituted with env var FOO From ef946dc4ec6e2b18ea7aa44c507125eb5918db8e Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 12:27:11 -0500 Subject: [PATCH 12/23] Add comments --- service/command.go | 1 - .../defaultconfigprovider.go | 60 ++++++++++++------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/service/command.go b/service/command.go index d35ef55894e..825b962868a 100644 --- a/service/command.go +++ b/service/command.go @@ -28,7 +28,6 @@ func NewCommand(set CollectorSettings) *cobra.Command { SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { if set.ConfigMapProvider == nil { - //set.ConfigMapProvider = configmapprovider.NewLocal(getConfigFlag(), getSetFlag()) set.ConfigMapProvider = defaultconfigprovider.NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) } col, err := New(set) diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 6c74c846965..0fea82f3c80 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -26,28 +26,43 @@ import ( ) type defaultConfigProvider struct { - initial configmapprovider.MapProvider - merged configmapprovider.MapProvider - factories component.Factories + localRoot configmapprovider.MapProvider + mergedRoot configmapprovider.MapProvider + factories component.Factories } +// NewDefaultConfigProvider creates a MapProvider that uses the local config file, +// the properties and the additional config sources specified in the config_sources +// and merge_configs sections to build the overall config map. +// +// It first loads config maps from all providers that are specified in the merge_configs +// section, merging them in the order they are specified (i.e. the last in the list has +// the highest precedence), then merges the local config from the file and properties, +// then performs substitution of all config values referenced using $ syntax. func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.MapProvider { localProvider := configmapprovider.NewLocal(configFileName, properties) - return &defaultConfigProvider{initial: localProvider, factories: factories} + return &defaultConfigProvider{localRoot: localProvider, factories: factories} } -func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(event *configmapprovider.ChangeEvent)) (configmapprovider.RetrievedMap, error) { - r, err := mp.initial.Retrieve(ctx, onChange) +func (mp *defaultConfigProvider) Retrieve( + ctx context.Context, + onChange func(event *configmapprovider.ChangeEvent), +) (configmapprovider.RetrievedMap, error) { + // Retrieve the local first. + r, err := mp.localRoot.Retrieve(ctx, onChange) if err != nil { return nil, err } - rootMap, err := r.Get(ctx) + localRootMap, err := r.Get(ctx) - configSources, mergeConfigs, err := unmarshalSources(ctx, rootMap, mp.factories) + // Unmarshal config sources. + configSources, mergeConfigs, err := unmarshalSources(ctx, localRootMap, mp.factories) if err != nil { return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err) } + // Iterate over all sources specified in the merge_configs section and create + // a provider for each. var rootProviders []configmapprovider.MapProvider for _, configSource := range mergeConfigs { configSourceID, err := config.NewComponentIDFromString(configSource) @@ -57,21 +72,24 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve configSource, ok := configSources[configSourceID] if !ok { - return nil, fmt.Errorf("config source %q must be defined in config_sources section", configSourceID) + return nil, fmt.Errorf("config source %q must be defined in config_sources section", configSource) } mapProvider, ok := configSource.(configmapprovider.MapProvider) if !ok { - return nil, fmt.Errorf("config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", configSourceID) + return nil, fmt.Errorf("config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", configSource) } rootProviders = append(rootProviders, mapProvider) } - rootProviders = append(rootProviders, configmapprovider.NewSimple(rootMap)) - mp.merged = configmapprovider.NewMerge(rootProviders...) + // Make the local root map the last (highest-precedence) root MapProvider. + rootProviders = append(rootProviders, configmapprovider.NewSimple(localRootMap)) + + // Create a merging provider and retrieve the config from it. + mp.mergedRoot = configmapprovider.NewMerge(rootProviders...) - retrieved, err := mp.merged.Retrieve(ctx, onChange) + retrieved, err := mp.mergedRoot.Retrieve(ctx, onChange) if err != nil { return nil, fmt.Errorf("cannot retrive the configuration: %w", err) } @@ -79,16 +97,17 @@ func (mp *defaultConfigProvider) Retrieve(ctx context.Context, onChange func(eve return &valueSubstitutor{onChange: onChange, retrieved: retrieved, configSources: configSources}, nil } +type rootConfig struct { + ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` + MergeConfigs []string `mapstructure:"merge_configs"` +} + func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( configSources map[config.ComponentID]configmapprovider.Provider, mergeConfigs []string, err error, ) { - type RootConfig struct { - ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` - MergeConfigs []string `mapstructure:"merge_configs"` - } - var rootCfg RootConfig + var rootCfg rootConfig err = rootMap.Unmarshal(&rootCfg) if err != nil { return nil, nil, err @@ -105,6 +124,7 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon } sourceType := id.Type() + // See if we have a factory for this config source type. factoryBase, ok := factories.ConfigSources[sourceType] if !ok { return nil, nil, fmt.Errorf("unknown config source type %q (did you register the config source factory?)", sourceType) @@ -136,8 +156,8 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon } func (mp *defaultConfigProvider) Shutdown(ctx context.Context) error { - if mp.merged != nil { - return mp.merged.Shutdown(ctx) + if mp.mergedRoot != nil { + return mp.mergedRoot.Shutdown(ctx) } return nil } From 05047ee3a550b00d9420703b3df600674a5122b4 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 12:30:53 -0500 Subject: [PATCH 13/23] Move ConfigSourceSettings to defaultunmarshaler --- config/configunmarshaler/defaultunmarshaler.go | 18 +++++++++++------- .../defaultconfigprovider.go | 7 +------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/config/configunmarshaler/defaultunmarshaler.go b/config/configunmarshaler/defaultunmarshaler.go index 93e1e6472d7..40cd84b74ce 100644 --- a/config/configunmarshaler/defaultunmarshaler.go +++ b/config/configunmarshaler/defaultunmarshaler.go @@ -65,14 +65,18 @@ const ( pipelinesKeyName = "pipelines" ) +type ConfigSourceSettings struct { + ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` + MergeConfigs []string `mapstructure:"merge_configs"` +} + type configSettings struct { - Receivers map[config.ComponentID]map[string]interface{} `mapstructure:"receivers"` - Processors map[config.ComponentID]map[string]interface{} `mapstructure:"processors"` - Exporters map[config.ComponentID]map[string]interface{} `mapstructure:"exporters"` - Extensions map[config.ComponentID]map[string]interface{} `mapstructure:"extensions"` - ConfigSources map[config.ComponentID]map[string]interface{} `mapstructure:"config_sources"` - MergeConfigs []string `mapstructure:"merge_configs"` - Service map[string]interface{} `mapstructure:"service"` + ConfigSourceSettings `mapstructure:",squash"` + Receivers map[config.ComponentID]map[string]interface{} `mapstructure:"receivers"` + Processors map[config.ComponentID]map[string]interface{} `mapstructure:"processors"` + Exporters map[config.ComponentID]map[string]interface{} `mapstructure:"exporters"` + Extensions map[config.ComponentID]map[string]interface{} `mapstructure:"extensions"` + Service map[string]interface{} `mapstructure:"service"` } type defaultUnmarshaler struct{} diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 0fea82f3c80..2954e51fb65 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -97,17 +97,12 @@ func (mp *defaultConfigProvider) Retrieve( return &valueSubstitutor{onChange: onChange, retrieved: retrieved, configSources: configSources}, nil } -type rootConfig struct { - ConfigSources []map[string]map[string]interface{} `mapstructure:"config_sources"` - MergeConfigs []string `mapstructure:"merge_configs"` -} - func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( configSources map[config.ComponentID]configmapprovider.Provider, mergeConfigs []string, err error, ) { - var rootCfg rootConfig + var rootCfg configunmarshaler.ConfigSourceSettings err = rootMap.Unmarshal(&rootCfg) if err != nil { return nil, nil, err From aca583eceb6ba1c3794648f96441cf8a0dc1ad8a Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 13:15:35 -0500 Subject: [PATCH 14/23] Add support for command line config sources --- service/collector_windows.go | 5 +- service/command.go | 6 +- .../defaultconfigprovider.go | 55 +++++++++++++++++- .../mapfromvalueprovider.go | 57 +++++++++++++++++++ 4 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 service/defaultconfigprovider/mapfromvalueprovider.go diff --git a/service/collector_windows.go b/service/collector_windows.go index 10f8ceda55d..7503d13db51 100644 --- a/service/collector_windows.go +++ b/service/collector_windows.go @@ -131,7 +131,10 @@ func openEventLog(serviceName string) (*eventlog.Log, error) { func newWithWindowsEventLogCore(set CollectorSettings, elog *eventlog.Log) (*Collector, error) { if set.ConfigMapProvider == nil { - set.ConfigMapProvider = NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) + set.ConfigMapProvider, err = NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) + if err != nil { + return nil, err + } } set.LoggingOptions = append( set.LoggingOptions, diff --git a/service/command.go b/service/command.go index 825b962868a..c92a93de403 100644 --- a/service/command.go +++ b/service/command.go @@ -28,7 +28,11 @@ func NewCommand(set CollectorSettings) *cobra.Command { SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { if set.ConfigMapProvider == nil { - set.ConfigMapProvider = defaultconfigprovider.NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) + var err error + set.ConfigMapProvider, err = defaultconfigprovider.NewDefaultConfigProvider(getConfigFlag(), getSetFlag(), set.Factories) + if err != nil { + return err + } } col, err := New(set) if err != nil { diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 2954e51fb65..70119405487 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -17,6 +17,7 @@ package defaultconfigprovider // import "go.opentelemetry.io/collector/config/co import ( "context" "fmt" + "strings" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configmapprovider" @@ -39,9 +40,54 @@ type defaultConfigProvider struct { // section, merging them in the order they are specified (i.e. the last in the list has // the highest precedence), then merges the local config from the file and properties, // then performs substitution of all config values referenced using $ syntax. -func NewDefaultConfigProvider(configFileName string, properties []string, factories component.Factories) configmapprovider.MapProvider { - localProvider := configmapprovider.NewLocal(configFileName, properties) - return &defaultConfigProvider{localRoot: localProvider, factories: factories} +func NewDefaultConfigProvider(configFlagValue string, properties []string, factories component.Factories) (configmapprovider.MapProvider, error) { + var rootProvider configmapprovider.MapProvider + + configFlagParts := strings.SplitN(configFlagValue, ":", 2) + if len(configFlagParts) == 1 { + rootProvider = configmapprovider.NewLocal(configFlagValue, properties) + } else { + var err error + rootProvider, err = createInlineMapProvider(configFlagParts[0], configFlagValue, factories) + if err != nil { + return nil, err + } + } + return &defaultConfigProvider{localRoot: rootProvider, factories: factories}, nil +} + +func createInlineMapProvider(sourceType string, configFlagValue string, factories component.Factories) (configmapprovider.MapProvider, error) { + cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(configFlagValue) + if err != nil { + return nil, fmt.Errorf("invalid format for --config flag value") + } + + factory, ok := factories.ConfigSources[config.Type(cfgSrcName)] + if !ok { + var allTypes []string + for t := range factories.ConfigSources { + allTypes = append(allTypes, string(t)) + } + return nil, fmt.Errorf("unknown source type %q (try one of %s)", sourceType, strings.Join(allTypes, ",")) + } + + cfg := factory.CreateDefaultConfig() + configSource, err := factory.CreateConfigSource(context.Background(), component.ConfigSourceCreateSettings{}, cfg) + if err != nil { + return nil, err + } + + valueProvider, ok := configSource.(configmapprovider.ValueProvider) + if !ok { + return nil, fmt.Errorf("config source %s cannot be used from command line because it is not a ValueProvider", sourceType) + } + + return &mapFromValueProvider{ + valueProvider: valueProvider, + selector: selector, + paramsConfigMap: paramsConfigMap, + configSourceName: sourceType, + }, nil } func (mp *defaultConfigProvider) Retrieve( @@ -54,6 +100,9 @@ func (mp *defaultConfigProvider) Retrieve( return nil, err } localRootMap, err := r.Get(ctx) + if err != nil { + return nil, err + } // Unmarshal config sources. configSources, mergeConfigs, err := unmarshalSources(ctx, localRootMap, mp.factories) diff --git a/service/defaultconfigprovider/mapfromvalueprovider.go b/service/defaultconfigprovider/mapfromvalueprovider.go new file mode 100644 index 00000000000..ca0f356fba9 --- /dev/null +++ b/service/defaultconfigprovider/mapfromvalueprovider.go @@ -0,0 +1,57 @@ +package defaultconfigprovider + +import ( + "bytes" + "context" + "fmt" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" +) + +type mapFromValueProvider struct { + configSourceName string + valueProvider configmapprovider.ValueProvider + selector string + paramsConfigMap *config.Map +} + +func (m mapFromValueProvider) Shutdown(ctx context.Context) error { + return m.valueProvider.Shutdown(ctx) +} + +func (m mapFromValueProvider) Retrieve( + ctx context.Context, onChange func(*configmapprovider.ChangeEvent), +) (configmapprovider.RetrievedMap, error) { + retrieved, err := m.valueProvider.Retrieve(ctx, onChange, m.selector, m.paramsConfigMap) + if err != nil { + return nil, err + } + return &mapFromValueRetrieved{retrieved: retrieved, configSourceName: m.configSourceName}, nil +} + +type mapFromValueRetrieved struct { + retrieved configmapprovider.RetrievedValue + configSourceName string +} + +func (m mapFromValueRetrieved) Get(ctx context.Context) (cfgMap *config.Map, err error) { + val, err := m.retrieved.Get(ctx) + + switch v := val.(type) { + case string: + cfgMap, err = config.NewMapFromBuffer(bytes.NewReader([]byte(v))) + case []byte: + cfgMap, err = config.NewMapFromBuffer(bytes.NewReader(v)) + case *config.Map: + cfgMap = v + default: + err = fmt.Errorf("config source %q returned invalid data (must return a string, []byte or a config.Map", m.configSourceName) + } + + return cfgMap, err +} + +func (m mapFromValueRetrieved) Close(ctx context.Context) error { + return m.retrieved.Close(ctx) +} From 87415b0d9d1df909f06f001cbef17ff3c9b1caf9 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 14:11:30 -0500 Subject: [PATCH 15/23] Add http config source --- configsource/http/config.go | 10 +++ configsource/http/configsource.go | 75 +++++++++++++++++++ configsource/http/factory.go | 37 +++++++++ .../mapfromvalueprovider.go | 3 + 4 files changed, 125 insertions(+) create mode 100644 configsource/http/config.go create mode 100644 configsource/http/configsource.go create mode 100644 configsource/http/factory.go diff --git a/configsource/http/config.go b/configsource/http/config.go new file mode 100644 index 00000000000..02a9c2da292 --- /dev/null +++ b/configsource/http/config.go @@ -0,0 +1,10 @@ +package http + +import ( + "go.opentelemetry.io/collector/config" +) + +// Config defines configuration for logging exporter. +type Config struct { + config.ConfigSourceSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct +} diff --git a/configsource/http/configsource.go b/configsource/http/configsource.go new file mode 100644 index 00000000000..b53e91ec45e --- /dev/null +++ b/configsource/http/configsource.go @@ -0,0 +1,75 @@ +package http + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" +) + +type configSource struct { + scheme string +} + +func (c configSource) Retrieve( + ctx context.Context, onChange func(*configmapprovider.ChangeEvent), selector string, paramsConfigMap *config.Map, +) (configmapprovider.RetrievedValue, error) { + return &retrieved{scheme: c.scheme, selector: selector, paramsConfigMap: paramsConfigMap}, nil +} + +func (c configSource) Shutdown(ctx context.Context) error { + return nil +} + +type retrieved struct { + scheme string + selector string + paramsConfigMap *config.Map +} + +func (r retrieved) Get(ctx context.Context) (interface{}, error) { + + // TODO: instead of passing scheme, selector and paramsConfigMap pass the entire + // invocation string and avoid re-creating the URL here. + + urlStr := r.scheme + ":" + r.selector + + if r.paramsConfigMap != nil { + queryVals := url.Values{} + for k, v := range r.paramsConfigMap.ToStringMap() { + str, ok := v.(string) + if !ok { + return nil, fmt.Errorf("invalid value for paramter %q", k) + } + queryVals.Set(k, str) + } + + query := queryVals.Encode() + if query != "" { + urlStr = urlStr + "?" + query + } + } + + resp, err := http.Get(urlStr) + if err != nil { + return nil, fmt.Errorf("cannot load config from %s: %w", urlStr, err) + } + defer resp.Body.Close() + + // TODO: handle HTTP response codes. + + bytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("cannot load config from %s: %w", urlStr, err) + } + + return bytes, nil +} + +func (r retrieved) Close(ctx context.Context) error { + return nil +} diff --git a/configsource/http/factory.go b/configsource/http/factory.go new file mode 100644 index 00000000000..c4924f124c5 --- /dev/null +++ b/configsource/http/factory.go @@ -0,0 +1,37 @@ +package http + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" + "go.opentelemetry.io/collector/internal/internalinterface" +) + +type factory struct { + internalinterface.BaseInternal + scheme string +} + +func (f factory) Type() config.Type { + return config.Type(f.scheme) +} + +func (f factory) CreateDefaultConfig() config.ConfigSource { + return &Config{ + ConfigSourceSettings: config.NewConfigSourceSettings(config.NewComponentID(f.Type())), + } +} + +func (f factory) CreateConfigSource(ctx context.Context, set component.ConfigSourceCreateSettings, cfg config.ConfigSource) (configmapprovider.Provider, error) { + return &configSource{scheme: f.scheme}, nil +} + +func NewHTTPFactory() component.ConfigSourceFactory { + return &factory{scheme: "http"} +} + +func NewHTTPSFactory() component.ConfigSourceFactory { + return &factory{scheme: "https"} +} diff --git a/service/defaultconfigprovider/mapfromvalueprovider.go b/service/defaultconfigprovider/mapfromvalueprovider.go index ca0f356fba9..b7d6068ed2c 100644 --- a/service/defaultconfigprovider/mapfromvalueprovider.go +++ b/service/defaultconfigprovider/mapfromvalueprovider.go @@ -37,6 +37,9 @@ type mapFromValueRetrieved struct { func (m mapFromValueRetrieved) Get(ctx context.Context) (cfgMap *config.Map, err error) { val, err := m.retrieved.Get(ctx) + if err != nil { + return nil, err + } switch v := val.(type) { case string: From f05015ff0472c9e5bf9d6d08fb2484513401d313 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 14:23:18 -0500 Subject: [PATCH 16/23] Minor refactoring --- .../defaultconfigprovider.go | 43 +++------------- .../inlinemapprovider.go | 51 +++++++++++++++++++ .../mapfromvalueprovider.go | 2 +- 3 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 service/defaultconfigprovider/inlinemapprovider.go diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 70119405487..b8f073c526b 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -45,49 +45,22 @@ func NewDefaultConfigProvider(configFlagValue string, properties []string, facto configFlagParts := strings.SplitN(configFlagValue, ":", 2) if len(configFlagParts) == 1 { - rootProvider = configmapprovider.NewLocal(configFlagValue, properties) + // The config flag value is just a config file name. + fileName := configFlagValue + rootProvider = configmapprovider.NewFile(fileName) } else { + // The provider is specified directly on the command line. Create it. var err error - rootProvider, err = createInlineMapProvider(configFlagParts[0], configFlagValue, factories) + rootProvider, err = createInlineMapProvider(configFlagValue, factories) if err != nil { return nil, err } } - return &defaultConfigProvider{localRoot: rootProvider, factories: factories}, nil -} - -func createInlineMapProvider(sourceType string, configFlagValue string, factories component.Factories) (configmapprovider.MapProvider, error) { - cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(configFlagValue) - if err != nil { - return nil, fmt.Errorf("invalid format for --config flag value") - } - - factory, ok := factories.ConfigSources[config.Type(cfgSrcName)] - if !ok { - var allTypes []string - for t := range factories.ConfigSources { - allTypes = append(allTypes, string(t)) - } - return nil, fmt.Errorf("unknown source type %q (try one of %s)", sourceType, strings.Join(allTypes, ",")) - } - cfg := factory.CreateDefaultConfig() - configSource, err := factory.CreateConfigSource(context.Background(), component.ConfigSourceCreateSettings{}, cfg) - if err != nil { - return nil, err - } + // Merge properties. + rootProvider = configmapprovider.NewMerge(rootProvider, configmapprovider.NewProperties(properties)) - valueProvider, ok := configSource.(configmapprovider.ValueProvider) - if !ok { - return nil, fmt.Errorf("config source %s cannot be used from command line because it is not a ValueProvider", sourceType) - } - - return &mapFromValueProvider{ - valueProvider: valueProvider, - selector: selector, - paramsConfigMap: paramsConfigMap, - configSourceName: sourceType, - }, nil + return &defaultConfigProvider{localRoot: rootProvider, factories: factories}, nil } func (mp *defaultConfigProvider) Retrieve( diff --git a/service/defaultconfigprovider/inlinemapprovider.go b/service/defaultconfigprovider/inlinemapprovider.go new file mode 100644 index 00000000000..2bee0bdb2e1 --- /dev/null +++ b/service/defaultconfigprovider/inlinemapprovider.go @@ -0,0 +1,51 @@ +package defaultconfigprovider + +import ( + "context" + "fmt" + "strings" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/config/configmapprovider" +) + +func createInlineMapProvider(configFlagValue string, factories component.Factories) (configmapprovider.MapProvider, error) { + // Parse the command line flag. + cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(configFlagValue) + if err != nil { + return nil, fmt.Errorf("invalid format for --config flag value") + } + + // Find the config source factory. + factory, ok := factories.ConfigSources[config.Type(cfgSrcName)] + if !ok { + var allTypes []string + for t := range factories.ConfigSources { + allTypes = append(allTypes, string(t)) + } + return nil, fmt.Errorf("unknown source type %q (try one of %s)", cfgSrcName, strings.Join(allTypes, ",")) + } + + // Create the config source. + cfg := factory.CreateDefaultConfig() + configSource, err := factory.CreateConfigSource(context.Background(), component.ConfigSourceCreateSettings{}, cfg) + if err != nil { + return nil, err + } + + valueProvider, ok := configSource.(configmapprovider.ValueProvider) + if !ok { + return nil, fmt.Errorf("config source %s cannot be used from command line because it is not a ValueProvider", cfgSrcName) + } + + // Convert retrieved value into a config map. + mapProvider := &mapFromValueProvider{ + valueProvider: valueProvider, + selector: selector, + paramsConfigMap: paramsConfigMap, + configSourceName: cfgSrcName, + } + + return mapProvider, nil +} diff --git a/service/defaultconfigprovider/mapfromvalueprovider.go b/service/defaultconfigprovider/mapfromvalueprovider.go index b7d6068ed2c..b4ee9ecfdaa 100644 --- a/service/defaultconfigprovider/mapfromvalueprovider.go +++ b/service/defaultconfigprovider/mapfromvalueprovider.go @@ -49,7 +49,7 @@ func (m mapFromValueRetrieved) Get(ctx context.Context) (cfgMap *config.Map, err case *config.Map: cfgMap = v default: - err = fmt.Errorf("config source %q returned invalid data (must return a string, []byte or a config.Map", m.configSourceName) + err = fmt.Errorf("config source %q returned invalid data (must return a string, []byte or *config.Map", m.configSourceName) } return cfgMap, err From 3d7fb5137c8d88b9af77affe7239702d705e725b Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 15:35:53 -0500 Subject: [PATCH 17/23] Make Retrive and RetriveValue different to allow implementing both --- config/configmapprovider/provider.go | 4 ++-- config/configmapprovider/valueprovider.go | 12 ++++++------ configsource/env/configsource.go | 2 +- configsource/http/configsource.go | 2 +- .../defaultconfigprovider/defaultconfigprovider.go | 4 ++++ .../defaultconfigprovider/mapfromvalueprovider.go | 2 +- service/defaultconfigprovider/valuesubstitutor.go | 2 +- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/config/configmapprovider/provider.go b/config/configmapprovider/provider.go index 271bd065c0e..c951cd15f4a 100644 --- a/config/configmapprovider/provider.go +++ b/config/configmapprovider/provider.go @@ -27,9 +27,9 @@ type Provider interface { // any resources that it may have created. // // This method must be called when the Collector service ends, either in case of - // success or error. Retrieve cannot be called after Shutdown. + // success or error. RetrieveValue cannot be called after Shutdown. // - // Should never be called concurrently with itself or with Retrieve. + // Should never be called concurrently with itself or with RetrieveValue. // If ctx is cancelled should return immediately with an error. Shutdown(ctx context.Context) error } diff --git a/config/configmapprovider/valueprovider.go b/config/configmapprovider/valueprovider.go index 93b410e8ca6..ba788e20192 100644 --- a/config/configmapprovider/valueprovider.go +++ b/config/configmapprovider/valueprovider.go @@ -26,7 +26,7 @@ import ( type ValueProvider interface { Provider - // Retrieve goes to the configuration source and retrieves the selected data which + // RetrieveValue goes to the configuration source and retrieves the selected data which // contains the value to be injected in the configuration and the corresponding watcher that // will be used to monitor for updates of the retrieved value. // @@ -38,22 +38,22 @@ type ValueProvider interface { // // If ctx is cancelled should return immediately with an error. // Should never be called concurrently with itself or with Shutdown. - Retrieve(ctx context.Context, onChange func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) + RetrieveValue(ctx context.Context, onChange func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) } -// RetrievedValue holds the result of a call to the Retrieve method of a ValueProvider object. +// RetrievedValue holds the result of a call to the RetrieveValue method of a ValueProvider object. // // The typical usage is the following: // -// r := valueProvider.Retrieve(selector, params) +// r := valueProvider.RetrieveValue(selector, params) // r.Get() // // wait for onChange() to be called. // r.Close() -// r = valueProvider.Retrieve(selector, params) +// r = valueProvider.RetrieveValue(selector, params) // r.Get() // // wait for onChange() to be called. // r.Close() -// // repeat Retrieve/Get/wait/Close cycle until it is time to shut down the Collector process. +// // repeat RetrieveValue/Get/wait/Close cycle until it is time to shut down the Collector process. // // ... // mapProvider.Shutdown() type RetrievedValue interface { diff --git a/configsource/env/configsource.go b/configsource/env/configsource.go index 5c5218f0b67..18dca5e7aea 100644 --- a/configsource/env/configsource.go +++ b/configsource/env/configsource.go @@ -11,7 +11,7 @@ import ( type configSource struct { } -func (c configSource) Retrieve( +func (c configSource) RetrieveValue( ctx context.Context, onChange func(*configmapprovider.ChangeEvent), selector string, paramsConfigMap *config.Map, ) (configmapprovider.RetrievedValue, error) { return &retrieved{selector: selector, paramsConfigMap: paramsConfigMap}, nil diff --git a/configsource/http/configsource.go b/configsource/http/configsource.go index b53e91ec45e..bc631e40f56 100644 --- a/configsource/http/configsource.go +++ b/configsource/http/configsource.go @@ -15,7 +15,7 @@ type configSource struct { scheme string } -func (c configSource) Retrieve( +func (c configSource) RetrieveValue( ctx context.Context, onChange func(*configmapprovider.ChangeEvent), selector string, paramsConfigMap *config.Map, ) (configmapprovider.RetrievedValue, error) { return &retrieved{scheme: c.scheme, selector: selector, paramsConfigMap: paramsConfigMap}, nil diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index b8f073c526b..8d0264087ad 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -156,6 +156,10 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon return nil, nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) } + if err := cfg.Validate(); err != nil { + return nil, nil, fmt.Errorf("config of config source %q is invalid: %w", sourceType, err) + } + factory, ok := factoryBase.(component.ConfigSourceFactory) if !ok { return nil, nil, fmt.Errorf("config source %q does not implement ConfigSourceFactory", sourceType) diff --git a/service/defaultconfigprovider/mapfromvalueprovider.go b/service/defaultconfigprovider/mapfromvalueprovider.go index b4ee9ecfdaa..ef9425e0bc9 100644 --- a/service/defaultconfigprovider/mapfromvalueprovider.go +++ b/service/defaultconfigprovider/mapfromvalueprovider.go @@ -23,7 +23,7 @@ func (m mapFromValueProvider) Shutdown(ctx context.Context) error { func (m mapFromValueProvider) Retrieve( ctx context.Context, onChange func(*configmapprovider.ChangeEvent), ) (configmapprovider.RetrievedMap, error) { - retrieved, err := m.valueProvider.Retrieve(ctx, onChange, m.selector, m.paramsConfigMap) + retrieved, err := m.valueProvider.RetrieveValue(ctx, onChange, m.selector, m.paramsConfigMap) if err != nil { return nil, err } diff --git a/service/defaultconfigprovider/valuesubstitutor.go b/service/defaultconfigprovider/valuesubstitutor.go index 72b1d8d2924..a34d30ed270 100644 --- a/service/defaultconfigprovider/valuesubstitutor.go +++ b/service/defaultconfigprovider/valuesubstitutor.go @@ -278,7 +278,7 @@ func (vp *valueSubstitutor) retrieveConfigSourceData(ctx context.Context, cfgSrc } } - retrieved, err := valueSrc.Retrieve(ctx, vp.onChange, selector, paramsConfigMap) + retrieved, err := valueSrc.RetrieveValue(ctx, vp.onChange, selector, paramsConfigMap) if err != nil { return nil, fmt.Errorf("config source %q failed to retrieve value: %w", cfgSrcName, err) } From 3fb2e337c591ac405903fa75390ca4fd1cb8864b Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 16:01:56 -0500 Subject: [PATCH 18/23] Allow using value config source in merge_configs section --- .../defaultconfigprovider.go | 94 +++++++++++++++---- 1 file changed, 78 insertions(+), 16 deletions(-) diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index 8d0264087ad..ab8b20cd1ab 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -43,8 +43,7 @@ type defaultConfigProvider struct { func NewDefaultConfigProvider(configFlagValue string, properties []string, factories component.Factories) (configmapprovider.MapProvider, error) { var rootProvider configmapprovider.MapProvider - configFlagParts := strings.SplitN(configFlagValue, ":", 2) - if len(configFlagParts) == 1 { + if !isValueConfigSourceRef(configFlagValue) { // The config flag value is just a config file name. fileName := configFlagValue rootProvider = configmapprovider.NewFile(fileName) @@ -63,6 +62,10 @@ func NewDefaultConfigProvider(configFlagValue string, properties []string, facto return &defaultConfigProvider{localRoot: rootProvider, factories: factories}, nil } +func isValueConfigSourceRef(configSourceRef string) bool { + return strings.Contains(configSourceRef, ":") +} + func (mp *defaultConfigProvider) Retrieve( ctx context.Context, onChange func(event *configmapprovider.ChangeEvent), @@ -86,22 +89,12 @@ func (mp *defaultConfigProvider) Retrieve( // Iterate over all sources specified in the merge_configs section and create // a provider for each. var rootProviders []configmapprovider.MapProvider - for _, configSource := range mergeConfigs { - configSourceID, err := config.NewComponentIDFromString(configSource) + for _, configSourceRef := range mergeConfigs { + mapProvider, err := mp.getConfigSourceMapProvider(configSourceRef, configSources) if err != nil { return nil, err } - configSource, ok := configSources[configSourceID] - if !ok { - return nil, fmt.Errorf("config source %q must be defined in config_sources section", configSource) - } - - mapProvider, ok := configSource.(configmapprovider.MapProvider) - if !ok { - return nil, fmt.Errorf("config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", configSource) - } - rootProviders = append(rootProviders, mapProvider) } @@ -113,10 +106,79 @@ func (mp *defaultConfigProvider) Retrieve( retrieved, err := mp.mergedRoot.Retrieve(ctx, onChange) if err != nil { - return nil, fmt.Errorf("cannot retrive the configuration: %w", err) + return nil, err } - return &valueSubstitutor{onChange: onChange, retrieved: retrieved, configSources: configSources}, nil + return &valueSubstitutor{ + onChange: onChange, retrieved: retrieved, configSources: configSources, + }, nil +} + +func (mp *defaultConfigProvider) getConfigSourceMapProvider( + configSourceRef string, configSources map[config.ComponentID]configmapprovider.Provider, +) (configmapprovider.MapProvider, error) { + if isValueConfigSourceRef(configSourceRef) { + cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(configSourceRef) + if err != nil { + return nil, err + } + + configSource, err := mp.findConfigSource(cfgSrcName, configSources) + if err != nil { + return nil, err + } + + valueProvider, ok := configSource.(configmapprovider.ValueProvider) + if !ok { + return nil, fmt.Errorf( + "config source %q cannot be used in merge_configs section since it does not implement ValueProvider interface", + configSource, + ) + } + + mapProvider := &mapFromValueProvider{ + valueProvider: valueProvider, + selector: selector, + paramsConfigMap: paramsConfigMap, + configSourceName: cfgSrcName, + } + return mapProvider, nil + + } else { + configSource, err := mp.findConfigSource(configSourceRef, configSources) + if err != nil { + return nil, err + } + + mapProvider, ok := configSource.(configmapprovider.MapProvider) + if !ok { + return nil, fmt.Errorf( + "config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", + configSource, + ) + } + return mapProvider, nil + } +} + +func (mp *defaultConfigProvider) findConfigSource( + cfgSrcName string, + configSources map[config.ComponentID]configmapprovider.Provider, +) (configmapprovider.Provider, error) { + configSourceID, err := config.NewComponentIDFromString(cfgSrcName) + if err != nil { + return nil, fmt.Errorf( + "invalid ID in config source specifier %q: %w", cfgSrcName, err, + ) + } + + configSource, ok := configSources[configSourceID] + if !ok { + return nil, fmt.Errorf( + "config source %q must be defined in config_sources section", cfgSrcName, + ) + } + return configSource, nil } func unmarshalSources(ctx context.Context, rootMap *config.Map, factories component.Factories) ( From 011c1d48c4b8d3e34d0922a396353008dec2d1ed Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 16:59:05 -0500 Subject: [PATCH 19/23] Add value config support to "file" source --- config/configmapprovider/file.go | 17 ++++++++++++++++- config/configmapprovider/simple.go | 13 +++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/config/configmapprovider/file.go b/config/configmapprovider/file.go index 88cc7b529cc..ccc7f19c0b7 100644 --- a/config/configmapprovider/file.go +++ b/config/configmapprovider/file.go @@ -27,7 +27,7 @@ type fileMapProvider struct { } // NewFile returns a new MapProvider that reads the configuration from the given file. -func NewFile(fileName string) MapProvider { +func NewFile(fileName string) *fileMapProvider { return &fileMapProvider{ fileName: fileName, } @@ -46,6 +46,21 @@ func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (R return &simpleRetrieved{confMap: cp}, nil } +func (fmp *fileMapProvider) RetrieveValue(_ context.Context, _ func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) { + if selector == "" { + return nil, errors.New("config file not specified") + } + + // TODO: support multiple file paths in selector (use some sort of delimiter). + + cp, err := config.NewMapFromFile(selector) + if err != nil { + return nil, fmt.Errorf("error loading config file %q: %w", fmp.fileName, err) + } + + return &simpleRetrievedValue{value: cp}, nil +} + func (*fileMapProvider) Shutdown(context.Context) error { return nil } diff --git a/config/configmapprovider/simple.go b/config/configmapprovider/simple.go index 4ec62ec8f98..045123776fc 100644 --- a/config/configmapprovider/simple.go +++ b/config/configmapprovider/simple.go @@ -49,3 +49,16 @@ func (sr *simpleRetrieved) Get(ctx context.Context) (*config.Map, error) { func (sr *simpleRetrieved) Close(ctx context.Context) error { return nil } + +// TODO: This probably will make sense to be exported, but needs better name and documentation. +type simpleRetrievedValue struct { + value interface{} +} + +func (sr *simpleRetrievedValue) Get(ctx context.Context) (interface{}, error) { + return sr.value, nil +} + +func (sr *simpleRetrievedValue) Close(ctx context.Context) error { + return nil +} From 86b69e4db8b2daa74ab281e49a3d12b2f7ba9f12 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 17:01:55 -0500 Subject: [PATCH 20/23] Delete unneeded Manager and Expand provider --- config/configmapprovider/expand.go | 92 --- config/configmapprovider/expand_test.go | 122 ---- config/configtest/configtest.go | 2 +- config/internal/configsource/manager.go | 684 ------------------ config/internal/configsource/manager_test.go | 706 ------------------- 5 files changed, 1 insertion(+), 1605 deletions(-) delete mode 100644 config/configmapprovider/expand.go delete mode 100644 config/configmapprovider/expand_test.go delete mode 100644 config/internal/configsource/manager.go delete mode 100644 config/internal/configsource/manager_test.go diff --git a/config/configmapprovider/expand.go b/config/configmapprovider/expand.go deleted file mode 100644 index 89f460e874c..00000000000 --- a/config/configmapprovider/expand.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configmapprovider // import "go.opentelemetry.io/collector/config/configmapprovider" - -import ( - "context" - "fmt" - "os" -) - -type expandMapProvider struct { - base MapProvider -} - -// NewExpand returns a MapProvider, that expands all environment variables for a -// config.Map provided by the given MapProvider. -func NewExpand(base MapProvider) MapProvider { - return &expandMapProvider{ - base: base, - } -} - -func (emp *expandMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { - retr, err := emp.base.Retrieve(ctx, onChange) - if err != nil { - return nil, fmt.Errorf("failed to retrieve from base provider: %w", err) - } - cfgMap, err := retr.Get(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get from base provider retrieved: %w", err) - } - for _, k := range cfgMap.AllKeys() { - cfgMap.Set(k, expandStringValues(cfgMap.Get(k))) - } - return &simpleRetrieved{confMap: cfgMap}, nil -} - -func (emp *expandMapProvider) Shutdown(ctx context.Context) error { - return emp.base.Shutdown(ctx) -} - -func expandStringValues(value interface{}) interface{} { - switch v := value.(type) { - default: - return v - case string: - return expandEnv(v) - case []interface{}: - nslice := make([]interface{}, 0, len(v)) - for _, vint := range v { - nslice = append(nslice, expandStringValues(vint)) - } - return nslice - case map[string]interface{}: - nmap := make(map[interface{}]interface{}, len(v)) - for k, vint := range v { - nmap[k] = expandStringValues(vint) - } - return nmap - case map[interface{}]interface{}: - nmap := make(map[interface{}]interface{}, len(v)) - for k, vint := range v { - nmap[k] = expandStringValues(vint) - } - return nmap - } -} - -func expandEnv(s string) string { - return os.Expand(s, func(str string) string { - // This allows escaping environment variable substitution via $$, e.g. - // - $FOO will be substituted with env var FOO - // - $$FOO will be replaced with $FOO - // - $$$FOO will be replaced with $ + substituted env var FOO - if str == "$" { - return "$" - } - return os.Getenv(str) - }) -} diff --git a/config/configmapprovider/expand_test.go b/config/configmapprovider/expand_test.go deleted file mode 100644 index 4d682335b8d..00000000000 --- a/config/configmapprovider/expand_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configmapprovider - -import ( - "context" - "errors" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "go.opentelemetry.io/collector/config" -) - -func TestBaseRetrieveFailsOnRetrieve(t *testing.T) { - retErr := errors.New("test error") - exp := NewExpand(&mockProvider{retrieveErr: retErr}) - defer exp.Shutdown(context.Background()) - _, err := exp.Retrieve(context.Background(), nil) - require.Error(t, err) - require.ErrorIs(t, err, retErr) -} - -func TestBaseRetrieveFailsOnGet(t *testing.T) { - getErr := errors.New("test error") - exp := NewExpand(&mockProvider{retrieved: &mockRetrieved{getErr: getErr}}) - defer exp.Shutdown(context.Background()) - _, err := exp.Retrieve(context.Background(), nil) - require.Error(t, err) - require.ErrorIs(t, err, getErr) -} - -func TestExpand(t *testing.T) { - var testCases = []struct { - name string // test case name (also file name containing config yaml) - }{ - {name: "expand-with-no-env.yaml"}, - {name: "expand-with-partial-env.yaml"}, - {name: "expand-with-all-env.yaml"}, - } - - const valueExtra = "some string" - const valueExtraMapValue = "some map value" - const valueExtraListElement = "some list value" - assert.NoError(t, os.Setenv("EXTRA", valueExtra)) - assert.NoError(t, os.Setenv("EXTRA_MAP_VALUE_1", valueExtraMapValue+"_1")) - assert.NoError(t, os.Setenv("EXTRA_MAP_VALUE_2", valueExtraMapValue+"_2")) - assert.NoError(t, os.Setenv("EXTRA_LIST_VALUE_1", valueExtraListElement+"_1")) - assert.NoError(t, os.Setenv("EXTRA_LIST_VALUE_2", valueExtraListElement+"_2")) - - defer func() { - assert.NoError(t, os.Unsetenv("EXTRA")) - assert.NoError(t, os.Unsetenv("EXTRA_MAP_VALUE")) - assert.NoError(t, os.Unsetenv("EXTRA_LIST_VALUE_1")) - }() - - expectedCfgMap, errExpected := config.NewMapFromFile(path.Join("testdata", "expand-with-no-env.yaml")) - require.NoError(t, errExpected, "Unable to get expected config") - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - // Retrieve the config - emp := NewExpand(NewFile(path.Join("testdata", test.name))) - cp, err := emp.Retrieve(context.Background(), nil) - require.NoError(t, err, "Unable to get config") - - // Test that expanded configs are the same with the simple config with no env vars. - m, err := cp.Get(context.Background()) - require.NoError(t, err) - assert.Equal(t, expectedCfgMap.ToStringMap(), m.ToStringMap()) - }) - } -} - -func TestExpand_EscapedEnvVars(t *testing.T) { - const receiverExtraMapValue = "some map value" - assert.NoError(t, os.Setenv("MAP_VALUE_2", receiverExtraMapValue)) - defer func() { - assert.NoError(t, os.Unsetenv("MAP_VALUE_2")) - }() - - // Retrieve the config - emp := NewExpand(NewFile(path.Join("testdata", "expand-escaped-env.yaml"))) - cp, err := emp.Retrieve(context.Background(), nil) - require.NoError(t, err, "Unable to get config") - - expectedMap := map[string]interface{}{ - "test_map": map[string]interface{}{ - // $$ -> escaped $ - "recv.1": "$MAP_VALUE_1", - // $$$ -> escaped $ + substituted env var - "recv.2": "$" + receiverExtraMapValue, - // $$$$ -> two escaped $ - "recv.3": "$$MAP_VALUE_3", - // escaped $ in the middle - "recv.4": "some${MAP_VALUE_4}text", - // $$$$ -> two escaped $ - "recv.5": "${ONE}${TWO}", - // trailing escaped $ - "recv.6": "text$", - // escaped $ alone - "recv.7": "$", - }} - m, err := cp.Get(context.Background()) - require.NoError(t, err) - assert.Equal(t, expectedMap, m.ToStringMap()) -} diff --git a/config/configtest/configtest.go b/config/configtest/configtest.go index 8cbedef61a8..7da32357e8a 100644 --- a/config/configtest/configtest.go +++ b/config/configtest/configtest.go @@ -35,7 +35,7 @@ var configFieldTagRegExp = regexp.MustCompile("^[a-z0-9][a-z0-9_]*$") // LoadConfig loads a config from file, and does NOT validate the configuration. func LoadConfig(fileName string, factories component.Factories) (*config.Config, error) { // Read yaml config from file - cp, err := configmapprovider.NewExpand(configmapprovider.NewFile(fileName)).Retrieve(context.Background(), nil) + cp, err := configmapprovider.NewFile(fileName).Retrieve(context.Background(), nil) if err != nil { return nil, err } diff --git a/config/internal/configsource/manager.go b/config/internal/configsource/manager.go deleted file mode 100644 index e8579a7f8e7..00000000000 --- a/config/internal/configsource/manager.go +++ /dev/null @@ -1,684 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configsource // import "go.opentelemetry.io/collector/config/internal/configsource" - -import ( - "bytes" - "context" - "errors" - "fmt" - "net/url" - "os" - "strings" - "sync" - - "github.com/spf13/cast" - "go.uber.org/multierr" - "gopkg.in/yaml.v2" - - "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/config/experimental/configsource" -) - -const ( - // configSourcesKey is the key used in YAML files to hold the config sources, and respective - // configurations that can be used in the configuration to be processed. - configSourcesKey = "config_sources" - // configSourceNameDelimChar is the char used to terminate the name of config source - // when it is used to retrieve values to inject in the configuration - configSourceNameDelimChar = ':' - // expandPrefixChar is the char used to prefix strings that can be expanded, - // either environment variables or config sources. - expandPrefixChar = '$' - // typeAndNameSeparator is the separator that is used between type and name in type/name - // composite keys. - typeAndNameSeparator = '/' -) - -// private error types to help with testability -type ( - errUnknownConfigSource struct{ error } -) - -// Manager is used to inject data from config sources into a configuration and also -// to monitor for updates on the items injected into the configuration. All methods -// of a Manager must be called only once and have an expected sequence: -// -// 1. NewManager to create a new instance; -// 2. Resolve to inject the data from config sources into a configuration; -// 3. WatchForUpdate in a goroutine to wait for configuration updates; -// 4. WaitForWatcher to wait until the watchers are in place; -// 5. Close to close the instance; -// -// The current syntax to reference a config source in a YAML is provisional. Currently -// single-line: -// -// param_to_be_retrieved: $:[?] -// -// bracketed single-line: -// -// param_to_be_retrieved: ${:[?]} -// -// and multi-line are supported: -// -// param_to_be_retrieved: | -// $: -// [] -// -// The is a name string used to identify the config source instance to be used -// to retrieve the value. -// -// The is the mandatory parameter required when retrieving data from a config source. -// -// Not all config sources need the optional parameters, they are used to provide extra control when -// retrieving and preparing the data to be injected into the configuration. -// -// For single-line format uses the same syntax as URL query parameters. -// Hypothetical example in a YAML file: -// -// component: -// config_field: $file:/etc/secret.bin?binary=true -// -// For multi-line format uses syntax as a YAML inside YAML. Possible usage -// example in a YAML file: -// -// component: -// config_field: | -// $yamltemplate: /etc/log_template.yaml -// logs_path: /var/logs/ -// timeout: 10s -// -// Not all config sources need these optional parameters, they are used to provide extra control when -// retrieving and data to be injected into the configuration. -// -// Assuming a config source named "env" that retrieve environment variables and one named "file" that -// retrieves contents from individual files, here are some examples: -// -// component: -// # Retrieves the value of the environment variable LOGS_DIR. -// logs_dir: $env:LOGS_DIR -// -// # Retrieves the value from the file /etc/secret.bin and injects its contents as a []byte. -// bytes_from_file: $file:/etc/secret.bin?binary=true -// -// # Retrieves the value from the file /etc/text.txt and injects its contents as a string. -// # Hypothetically the "file" config source by default tries to inject the file contents -// # as a string if params doesn't specify that "binary" is true. -// text_from_file: $file:/etc/text.txt -// -// Bracketed single-line should be used when concatenating a suffix to the value retrieved by -// the config source. Example: -// -// component: -// # Retrieves the value of the environment variable LOGS_DIR and appends /component.log to it. -// log_file_fullname: ${env:LOGS_DIR}/component.log -// -// Environment variables are expanded before passed to the config source when used in the selector or -// the optional parameters. Example: -// -// component: -// # Retrieves the value from the file text.txt located on the path specified by the environment -// # variable DATA_PATH. The name of the environment variable is the string after the delimiter -// # until the first character different than '_' and non-alpha-numeric. -// text_from_file: $file:$DATA_PATH/text.txt -// -// Since environment variables and config sources both use the '$', with or without brackets, as a prefix -// for their expansion it is necessary to have a way to distinguish between them. For the non-bracketed -// syntax the code will peek at the first character other than alpha-numeric and '_' after the '$'. If -// that character is a ':' it will treat it as a config source and as environment variable otherwise. -// For example: -// -// component: -// field_0: $PATH:/etc/logs # Injects the data from a config sourced named "PATH" using the selector "/etc/logs". -// field_1: $PATH/etc/logs # Expands the environment variable "PATH" and adds the suffix "/etc/logs" to it. -// -// So if you need to include an environment followed by ':' the bracketed syntax must be used instead: -// -// component: -// field_0: ${PATH}:/etc/logs # Expands the environment variable "PATH" and adds the suffix ":/etc/logs" to it. -// -// For the bracketed syntax the presence of ':' inside the brackets indicates that code will treat the bracketed -// contents as a config source. For example: -// -// component: -// field_0: ${file:/var/secret.txt} # Injects the data from a config sourced named "file" using the selector "/var/secret.txt". -// field_1: ${file}:/var/secret.txt # Expands the environment variable "file" and adds the suffix ":/var/secret.txt" to it. -// -// If the character following the '$' is in the set {'*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'} -// the code will consider it to be the name of an environment variable to expand, or config source if followed by ':'. Do not use any of these -// characters as the first char on the name of a config source or an environment variable (even if allowed by the system) to avoid unexpected -// results. -// -// For an overview about the internals of the Manager refer to the package README.md. -type Manager struct { - // configSources is map from ConfigSource names (as defined in the configuration) - // and the respective instances. - configSources map[string]configsource.ConfigSource - // watchers keeps track of all WatchForUpdate functions for retrieved values. - watchers []configsource.Watchable - // watchersWG is used to ensure that Close waits for all WatchForUpdate calls - // to complete. - watchersWG sync.WaitGroup - // watchingCh is used to notify users of the Manager that the WatchForUpdate function - // is ready and waiting for notifications. - watchingCh chan struct{} - // closeCh is used to notify the Manager WatchForUpdate function that the manager - // is being closed. - closeCh chan struct{} -} - -// NewManager creates a new instance of a Manager to be used to inject data from -// ConfigSource objects into a configuration and watch for updates on the injected -// data. -func NewManager(_ *config.Map) (*Manager, error) { - // TODO: Config sources should be extracted for the config itself, need Factories for that. - - return &Manager{ - watchingCh: make(chan struct{}), - closeCh: make(chan struct{}), - }, nil -} - -// Resolve inspects the given config.Map and resolves all config sources referenced -// in the configuration, returning a config.Map in which all env vars and config sources on -// the given input config map are resolved to actual literal values of the env vars or config sources. -// This method must be called only once per lifetime of a Manager object. -func (m *Manager) Resolve(ctx context.Context, configMap *config.Map) (*config.Map, error) { - res := config.NewMap() - allKeys := configMap.AllKeys() - for _, k := range allKeys { - if strings.HasPrefix(k, configSourcesKey) { - // Remove everything under the config_sources section. The `config_sources` section - // is read when loading the config sources used in the configuration, but it is not - // part of the resulting configuration returned via *config.Map. - continue - } - - value, err := m.parseConfigValue(ctx, configMap.Get(k)) - if err != nil { - return nil, err - } - res.Set(k, value) - } - - return res, nil -} - -// WatchForUpdate must watch for updates on any of the values retrieved from config sources -// and injected into the configuration. Typically this method is launched in a goroutine, the -// method WaitForWatcher blocks until the WatchForUpdate goroutine is running and ready. -func (m *Manager) WatchForUpdate() error { - // Use a channel to capture the first error returned by any watcher and another one - // to ensure completion of any remaining watcher also trying to report an error. - errChannel := make(chan error, 1) - doneCh := make(chan struct{}) - defer close(doneCh) - - for i := range m.watchers { - watcher := m.watchers[i] - m.watchersWG.Add(1) - go func() { - defer m.watchersWG.Done() - err := watcher.WatchForUpdate() - switch { - case errors.Is(err, configsource.ErrSessionClosed): - // The Session from which this watcher was retrieved is being closed. - // There is no error to report, just exit from the goroutine. - return - default: - select { - case errChannel <- err: - // Try to report any other error. - case <-doneCh: - // There was either one error published or the watcher was closed. - // This channel was closed and any goroutines waiting on these - // should simply close. - } - } - }() - } - - // All goroutines were created, they may not be running yet, but the manager WatchForUpdate - // is only waiting for any of the watchers to terminate. - close(m.watchingCh) - - select { - case err := <-errChannel: - // Return the first error that reaches the channel and ignore any other error. - return err - case <-m.closeCh: - // This covers the case that all watchers returned ErrWatcherNotSupported. - return configsource.ErrSessionClosed - } -} - -// WaitForWatcher blocks until the watchers used by WatchForUpdate are all ready. -// This is used to ensure that the watchers are in place before proceeding. -func (m *Manager) WaitForWatcher() { - <-m.watchingCh -} - -// Close terminates the WatchForUpdate function and closes all Session objects used -// in the configuration. It should be called -func (m *Manager) Close(ctx context.Context) error { - var errs error - for _, source := range m.configSources { - errs = multierr.Append(errs, source.Close(ctx)) - } - - close(m.closeCh) - m.watchersWG.Wait() - - return errs -} - -// parseConfigValue takes the value of a "config node" and process it recursively. The processing consists -// in transforming invocations of config sources and/or environment variables into literal data that can be -// used directly from a `config.Map` object. -func (m *Manager) parseConfigValue(ctx context.Context, value interface{}) (interface{}, error) { - switch v := value.(type) { - case string: - // Only if the value of the node is a string it can contain an env var or config source - // invocation that requires transformation. - return m.parseStringValue(ctx, v) - case []interface{}: - // The value is of type []interface{} when an array is used in the configuration, YAML example: - // - // array0: - // - elem0 - // - elem1 - // array1: - // - entry: - // str: elem0 - // - entry: - // str: $tstcfgsrc:elem1 - // - // Both "array0" and "array1" are going to be leaf config nodes hitting this case. - nslice := make([]interface{}, 0, len(v)) - for _, vint := range v { - value, err := m.parseConfigValue(ctx, vint) - if err != nil { - return nil, err - } - nslice = append(nslice, value) - } - return nslice, nil - case map[string]interface{}: - // The value is of type map[string]interface{} when an array in the configuration is populated with map - // elements. From the case above (for type []interface{}) each element of "array1" is going to hit the - // the current case block. - nmap := make(map[interface{}]interface{}, len(v)) - for k, vint := range v { - value, err := m.parseConfigValue(ctx, vint) - if err != nil { - return nil, err - } - nmap[k] = value - } - return nmap, nil - default: - // All other literals (int, boolean, etc) can't be further expanded so just return them as they are. - return v, nil - } -} - -// parseStringValue transforms environment variables and config sources, if any are present, on -// the given string in the configuration into an object to be inserted into the resulting configuration. -func (m *Manager) parseStringValue(ctx context.Context, s string) (interface{}, error) { - // Code based on os.Expand function. All delimiters that are checked against are - // ASCII so bytes are fine for this operation. - var buf []byte - - // Using i, j, and w variables to keep correspondence with os.Expand code. - // i tracks the index in s from which a slice to be appended to buf should start. - // j tracks the char being currently checked and also the end of the slice to be appended to buf. - // w tracks the number of characters being consumed after a prefix identifying env vars or config sources. - i := 0 - for j := 0; j < len(s); j++ { - // Skip chars until a candidate for expansion is found. - if s[j] == expandPrefixChar && j+1 < len(s) { - if buf == nil { - // Assuming that the length of the string will double after expansion of env vars and config sources. - buf = make([]byte, 0, 2*len(s)) - } - - // Append everything consumed up to the prefix char (but not including the prefix char) to the result. - buf = append(buf, s[i:j]...) - - var expandableContent, cfgSrcName string - w := 0 // number of bytes consumed on this pass - - switch { - case s[j+1] == expandPrefixChar: - // Escaping the prefix so $$ becomes a single $ without attempting - // to treat the string after it as a config source or env var. - expandableContent = string(expandPrefixChar) - w = 1 // consumed a single char - - case s[j+1] == '{': - // Bracketed usage, consume everything until first '}' exactly as os.Expand. - expandableContent, w = scanToClosingBracket(s[j+1:]) - expandableContent = strings.Trim(expandableContent, " ") // Allow for some spaces. - delimIndex := strings.Index(expandableContent, string(configSourceNameDelimChar)) - if len(expandableContent) > 1 && delimIndex > -1 { - // Bracket expandableContent contains ':' treating it as a config source. - cfgSrcName = expandableContent[:delimIndex] - } - - default: - // Non-bracketed usage, ie.: found the prefix char, it can be either a config - // source or an environment variable. - var name string - name, w = getTokenName(s[j+1:]) - expandableContent = name // Assume for now that it is an env var. - - // Peek next char after name, if it is a config source name delimiter treat the remaining of the - // string as a config source. - possibleDelimCharIndex := j + w + 1 - if possibleDelimCharIndex < len(s) && s[possibleDelimCharIndex] == configSourceNameDelimChar { - // This is a config source, since it is not delimited it will consume until end of the string. - cfgSrcName = name - expandableContent = s[j+1:] - w = len(expandableContent) // Set consumed bytes to the length of expandableContent - } - } - - // At this point expandableContent contains a string to be expanded, evaluate and expand it. - switch { - case cfgSrcName == "": - // Not a config source, expand as os.ExpandEnv - buf = osExpandEnv(buf, expandableContent, w) - - default: - // A config source, retrieve and apply results. - retrieved, err := m.retrieveConfigSourceData(ctx, cfgSrcName, expandableContent) - if err != nil { - return nil, err - } - - consumedAll := j+w+1 == len(s) - if consumedAll && len(buf) == 0 { - // This is the only expandableContent on the string, config - // source is free to return interface{} but parse it as YAML - // if it is a string or byte slice. - switch value := retrieved.(type) { - case []byte: - if err := yaml.Unmarshal(value, &retrieved); err != nil { - // The byte slice is an invalid YAML keep the original. - retrieved = value - } - case string: - if err := yaml.Unmarshal([]byte(value), &retrieved); err != nil { - // The string is an invalid YAML keep it as the original. - retrieved = value - } - } - - if mapIFace, ok := retrieved.(map[interface{}]interface{}); ok { - // yaml.Unmarshal returns map[interface{}]interface{} but config - // map uses map[string]interface{}, fix it with a cast. - retrieved = cast.ToStringMap(mapIFace) - } - - return retrieved, nil - } - - // Either there was a prefix already or there are still characters to be processed. - if retrieved == nil { - // Since this is going to be concatenated to a string use "" instead of nil, - // otherwise the string will end up with "". - retrieved = "" - } - - buf = append(buf, fmt.Sprintf("%v", retrieved)...) - } - - j += w // move the index of the char being checked (j) by the number of characters consumed (w) on this iteration. - i = j + 1 // update start index (i) of next slice of bytes to be copied. - } - } - - if buf == nil { - // No changes to original string, just return it. - return s, nil - } - - // Return whatever was accumulated on the buffer plus the remaining of the original string. - return string(buf) + s[i:], nil -} - -// retrieveConfigSourceData retrieves data from the specified config source and injects them into -// the configuration. The Manager tracks sessions and watcher objects as needed. -func (m *Manager) retrieveConfigSourceData(ctx context.Context, cfgSrcName, cfgSrcInvocation string) (interface{}, error) { - cfgSrc, ok := m.configSources[cfgSrcName] - if !ok { - return nil, newErrUnknownConfigSource(cfgSrcName) - } - - cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(cfgSrcInvocation) - if err != nil { - return nil, err - } - - // Recursively expand the selector. - var expandedSelector interface{} - expandedSelector, err = m.parseStringValue(ctx, selector) - if err != nil { - return nil, fmt.Errorf("failed to process selector for config source %q selector %q: %w", cfgSrcName, selector, err) - } - if selector, ok = expandedSelector.(string); !ok { - return nil, fmt.Errorf("processed selector must be a string instead got a %T %v", expandedSelector, expandedSelector) - } - - // Recursively resolve/parse any config source on the parameters. - if paramsConfigMap != nil { - paramsConfigMap, err = m.Resolve(ctx, paramsConfigMap) - if err != nil { - return nil, fmt.Errorf("failed to process parameters for config source %q invocation %q: %w", cfgSrcName, cfgSrcInvocation, err) - } - } - - retrieved, err := cfgSrc.Retrieve(ctx, selector, paramsConfigMap) - if err != nil { - return nil, fmt.Errorf("config source %q failed to retrieve value: %w", cfgSrcName, err) - } - - if watcher, ok := retrieved.(configsource.Watchable); ok { - m.watchers = append(m.watchers, watcher) - } - - return retrieved.Value(), nil -} - -func newErrUnknownConfigSource(cfgSrcName string) error { - return &errUnknownConfigSource{ - fmt.Errorf(`config source %q not found if this was intended to be an environment variable use "${%s}" instead"`, cfgSrcName, cfgSrcName), - } -} - -// parseCfgSrcInvocation parses the original string in the configuration that has a config source -// retrieve operation and return its "logical components": the config source name, the selector, and -// a config.Map to be used in this invocation of the config source. See Test_parseCfgSrcInvocation -// for some examples of input and output. -// The caller should check for error explicitly since it is possible for the -// other values to have been partially set. -func parseCfgSrcInvocation(s string) (cfgSrcName, selector string, paramsConfigMap *config.Map, err error) { - parts := strings.SplitN(s, string(configSourceNameDelimChar), 2) - if len(parts) != 2 { - err = fmt.Errorf("invalid config source syntax at %q, it must have at least the config source name and a selector", s) - return - } - cfgSrcName = strings.Trim(parts[0], " ") - - // Separate multi-line and single line case. - afterCfgSrcName := parts[1] - switch { - case strings.Contains(afterCfgSrcName, "\n"): - // Multi-line, until the first \n it is the selector, everything after as YAML. - parts = strings.SplitN(afterCfgSrcName, "\n", 2) - selector = strings.Trim(parts[0], " ") - - if len(parts) > 1 && len(parts[1]) > 0 { - var cp *config.Map - cp, err = config.NewMapFromBuffer(bytes.NewReader([]byte(parts[1]))) - if err != nil { - return - } - paramsConfigMap = cp - } - - default: - // Single line, and parameters as URL query. - const selectorDelim string = "?" - parts = strings.SplitN(parts[1], selectorDelim, 2) - selector = strings.Trim(parts[0], " ") - - if len(parts) == 2 { - paramsPart := parts[1] - paramsConfigMap, err = parseParamsAsURLQuery(paramsPart) - if err != nil { - err = fmt.Errorf("invalid parameters syntax at %q: %w", s, err) - return - } - } - } - - return cfgSrcName, selector, paramsConfigMap, err -} - -func parseParamsAsURLQuery(s string) (*config.Map, error) { - values, err := url.ParseQuery(s) - if err != nil { - return nil, err - } - - // Transform single array values in scalars. - params := make(map[string]interface{}) - for k, v := range values { - switch len(v) { - case 0: - params[k] = nil - case 1: - var iface interface{} - if err = yaml.Unmarshal([]byte(v[0]), &iface); err != nil { - return nil, err - } - params[k] = iface - default: - // It is a slice add element by element - elemSlice := make([]interface{}, 0, len(v)) - for _, elem := range v { - var iface interface{} - if err = yaml.Unmarshal([]byte(elem), &iface); err != nil { - return nil, err - } - elemSlice = append(elemSlice, iface) - } - params[k] = elemSlice - } - } - return config.NewMapFromStringMap(params), err -} - -// osExpandEnv replicate the internal behavior of os.ExpandEnv when handling env -// vars updating the buffer accordingly. -func osExpandEnv(buf []byte, name string, w int) []byte { - switch { - case name == "" && w > 0: - // Encountered invalid syntax; eat the - // characters. - case name == "" || name == "$": - // Valid syntax, but $ was not followed by a - // name. Leave the dollar character untouched. - buf = append(buf, expandPrefixChar) - default: - buf = append(buf, os.Getenv(name)...) - } - - return buf -} - -// scanToClosingBracket consumes everything until a closing bracket '}' following the -// same logic of function getShellName (os package, env.go) when handling environment -// variables with the "${}" syntax. It returns the expression between brackets -// and the number of characters consumed from the original string. -func scanToClosingBracket(s string) (string, int) { - for i := 1; i < len(s); i++ { - if s[i] == '}' { - if i == 1 { - return "", 2 // Bad syntax; eat "${}" - } - return s[1:i], i + 1 - } - } - return "", 1 // Bad syntax; eat "${" -} - -// getTokenName consumes characters until it has the name of either an environment -// variable or config source. It returns the name of the config source or environment -// variable and the number of characters consumed from the original string. -func getTokenName(s string) (string, int) { - if len(s) > 0 && isShellSpecialVar(s[0]) { - // Special shell character, treat it os.Expand function. - return s[0:1], 1 - } - - var i int - firstNameSepIdx := -1 - for i = 0; i < len(s); i++ { - if isAlphaNum(s[i]) { - // Continue while alphanumeric plus underscore. - continue - } - - if s[i] == typeAndNameSeparator && firstNameSepIdx == -1 { - // If this is the first type name separator store the index and continue. - firstNameSepIdx = i - continue - } - - // It is one of the following cases: - // 1. End of string - // 2. Reached a non-alphanumeric character, preceded by at most one - // typeAndNameSeparator character. - break - } - - if firstNameSepIdx != -1 && (i >= len(s) || s[i] != configSourceNameDelimChar) { - // Found a second non alpha-numeric character before the end of the string - // but it is not the config source delimiter. Use the name until the first - // name delimiter. - return s[:firstNameSepIdx], firstNameSepIdx - } - - return s[:i], i -} - -// Below are helper functions used by os.Expand, copied without changes from original sources (env.go). - -// isShellSpecialVar reports whether the character identifies a special -// shell variable such as $*. -func isShellSpecialVar(c uint8) bool { - switch c { - case '*', '#', '$', '@', '!', '?', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - return true - } - return false -} - -// isAlphaNum reports whether the byte is an ASCII letter, number, or underscore -func isAlphaNum(c uint8) bool { - return c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' -} diff --git a/config/internal/configsource/manager_test.go b/config/internal/configsource/manager_test.go deleted file mode 100644 index 95fdb3dae07..00000000000 --- a/config/internal/configsource/manager_test.go +++ /dev/null @@ -1,706 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configsource - -import ( - "context" - "errors" - "fmt" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/config/experimental/configsource" -) - -func TestConfigSourceManager_Simple(t *testing.T) { - ctx := context.Background() - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "test_selector": {Value: "test_value"}, - }, - }, - }) - - originalCfg := map[string]interface{}{ - "top0": map[string]interface{}{ - "int": 1, - "cfgsrc": "$tstcfgsrc:test_selector", - }, - } - expectedCfg := map[string]interface{}{ - "top0": map[string]interface{}{ - "int": 1, - "cfgsrc": "test_value", - }, - } - - cp := config.NewMapFromStringMap(originalCfg) - - res, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - assert.Equal(t, expectedCfg, res.ToStringMap()) - - doneCh := make(chan struct{}) - var errWatcher error - go func() { - defer close(doneCh) - errWatcher = manager.WatchForUpdate() - }() - - manager.WaitForWatcher() - assert.NoError(t, manager.Close(ctx)) - <-doneCh - assert.ErrorIs(t, errWatcher, configsource.ErrSessionClosed) -} - -func TestConfigSourceManager_ResolveRemoveConfigSourceSection(t *testing.T) { - cfg := map[string]interface{}{ - "config_sources": map[string]interface{}{ - "testcfgsrc": nil, - }, - "another_section": map[string]interface{}{ - "int": 42, - }, - } - - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{}, - }) - - res, err := manager.Resolve(context.Background(), config.NewMapFromStringMap(cfg)) - require.NoError(t, err) - require.NotNil(t, res) - - delete(cfg, "config_sources") - assert.Equal(t, cfg, res.ToStringMap()) -} - -func TestConfigSourceManager_ResolveErrors(t *testing.T) { - ctx := context.Background() - testErr := errors.New("test error") - - tests := []struct { - config map[string]interface{} - configSourceMap map[string]configsource.ConfigSource - name string - }{ - { - name: "incorrect_cfgsrc_ref", - config: map[string]interface{}{ - "cfgsrc": "$tstcfgsrc:selector?{invalid}", - }, - configSourceMap: map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{}, - }, - }, - { - name: "error_on_retrieve", - config: map[string]interface{}{ - "cfgsrc": "$tstcfgsrc:selector", - }, - configSourceMap: map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ErrOnRetrieve: testErr}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - manager := newManager(tt.configSourceMap) - - res, err := manager.Resolve(ctx, config.NewMapFromStringMap(tt.config)) - require.Error(t, err) - require.Nil(t, res) - require.NoError(t, manager.Close(ctx)) - }) - } -} - -func TestConfigSourceManager_YAMLInjection(t *testing.T) { - ctx := context.Background() - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "valid_yaml_str": {Value: ` -bool: true -int: 42 -source: string -map: - k0: v0 - k1: v1 -`}, - "invalid_yaml_str": {Value: ":"}, - "valid_yaml_byte_slice": {Value: []byte(` -bool: true -int: 42 -source: "[]byte" -map: - k0: v0 - k1: v1 -`)}, - "invalid_yaml_byte_slice": {Value: []byte(":")}, - }, - }, - }) - - file := path.Join("testdata", "yaml_injection.yaml") - cp, err := config.NewMapFromFile(file) - require.NoError(t, err) - - expectedFile := path.Join("testdata", "yaml_injection_expected.yaml") - expectedConfigMap, err := config.NewMapFromFile(expectedFile) - require.NoError(t, err) - expectedCfg := expectedConfigMap.ToStringMap() - - res, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - actualCfg := res.ToStringMap() - assert.Equal(t, expectedCfg, actualCfg) - assert.NoError(t, manager.Close(ctx)) -} - -func TestConfigSourceManager_ArraysAndMaps(t *testing.T) { - ctx := context.Background() - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "elem0": {Value: "elem0_value"}, - "elem1": {Value: "elem1_value"}, - "k0": {Value: "k0_value"}, - "k1": {Value: "k1_value"}, - }, - }, - }) - - file := path.Join("testdata", "arrays_and_maps.yaml") - cp, err := config.NewMapFromFile(file) - require.NoError(t, err) - - expectedFile := path.Join("testdata", "arrays_and_maps_expected.yaml") - expectedConfigMap, err := config.NewMapFromFile(expectedFile) - require.NoError(t, err) - - res, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - assert.Equal(t, expectedConfigMap.ToStringMap(), res.ToStringMap()) - assert.NoError(t, manager.Close(ctx)) -} - -func TestConfigSourceManager_ParamsHandling(t *testing.T) { - ctx := context.Background() - tstCfgSrc := testConfigSource{ - ValueMap: map[string]valueEntry{ - "elem0": {Value: nil}, - "elem1": { - Value: map[string]interface{}{ - "p0": true, - "p1": "a string with spaces", - "p3": 42, - }, - }, - "k0": {Value: nil}, - "k1": { - Value: map[string]interface{}{ - "p0": true, - "p1": "a string with spaces", - "p2": map[string]interface{}{ - "p2_0": "a nested map0", - "p2_1": true, - }, - }, - }, - }, - } - - // Set OnRetrieve to check if the parameters were parsed as expected. - tstCfgSrc.OnRetrieve = func(ctx context.Context, selector string, paramsConfigMap *config.Map) error { - paramsValue := (interface{})(nil) - if paramsConfigMap != nil { - paramsValue = paramsConfigMap.ToStringMap() - } - assert.Equal(t, tstCfgSrc.ValueMap[selector].Value, paramsValue) - return nil - } - - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &tstCfgSrc, - }) - - file := path.Join("testdata", "params_handling.yaml") - cp, err := config.NewMapFromFile(file) - require.NoError(t, err) - - expectedFile := path.Join("testdata", "params_handling_expected.yaml") - expectedConfigMap, err := config.NewMapFromFile(expectedFile) - require.NoError(t, err) - - res, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - assert.Equal(t, expectedConfigMap.ToStringMap(), res.ToStringMap()) - assert.NoError(t, manager.Close(ctx)) -} - -func TestConfigSourceManager_WatchForUpdate(t *testing.T) { - ctx := context.Background() - watchForUpdateCh := make(chan error, 1) - - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "test_selector": { - Value: "test_value", - WatchForUpdateFn: func() error { - return <-watchForUpdateCh - }, - }, - }, - }, - }) - - originalCfg := map[string]interface{}{ - "top0": map[string]interface{}{ - "var0": "$tstcfgsrc:test_selector", - }, - } - - cp := config.NewMapFromStringMap(originalCfg) - _, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - - doneCh := make(chan struct{}) - var errWatcher error - go func() { - defer close(doneCh) - errWatcher = manager.WatchForUpdate() - }() - - manager.WaitForWatcher() - watchForUpdateCh <- configsource.ErrValueUpdated - - <-doneCh - assert.ErrorIs(t, errWatcher, configsource.ErrValueUpdated) - assert.NoError(t, manager.Close(ctx)) -} - -func TestConfigSourceManager_MultipleWatchForUpdate(t *testing.T) { - ctx := context.Background() - - watchDoneCh := make(chan struct{}) - const watchForUpdateChSize int = 2 - watchForUpdateCh := make(chan error, watchForUpdateChSize) - watchForUpdateFn := func() error { - select { - case errFromWatchForUpdate := <-watchForUpdateCh: - return errFromWatchForUpdate - case <-watchDoneCh: - return configsource.ErrSessionClosed - } - } - - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "test_selector": { - Value: "test_value", - WatchForUpdateFn: watchForUpdateFn, - }, - }, - }, - }) - - originalCfg := map[string]interface{}{ - "top0": map[string]interface{}{ - "var0": "$tstcfgsrc:test_selector", - "var1": "$tstcfgsrc:test_selector", - "var2": "$tstcfgsrc:test_selector", - "var3": "$tstcfgsrc:test_selector", - }, - } - - cp := config.NewMapFromStringMap(originalCfg) - _, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - - doneCh := make(chan struct{}) - var errWatcher error - go func() { - defer close(doneCh) - errWatcher = manager.WatchForUpdate() - }() - - manager.WaitForWatcher() - - for i := 0; i < watchForUpdateChSize; i++ { - watchForUpdateCh <- configsource.ErrValueUpdated - } - - <-doneCh - assert.ErrorIs(t, errWatcher, configsource.ErrValueUpdated) - close(watchForUpdateCh) - assert.NoError(t, manager.Close(ctx)) -} - -func TestConfigSourceManager_EnvVarHandling(t *testing.T) { - require.NoError(t, os.Setenv("envvar", "envvar_value")) - defer func() { - assert.NoError(t, os.Unsetenv("envvar")) - }() - - ctx := context.Background() - tstCfgSrc := testConfigSource{ - ValueMap: map[string]valueEntry{ - "int_key": {Value: 42}, - }, - } - - // Intercept "params_key" and create an entry with the params themselves. - tstCfgSrc.OnRetrieve = func(ctx context.Context, selector string, paramsConfigMap *config.Map) error { - if selector == "params_key" { - tstCfgSrc.ValueMap[selector] = valueEntry{Value: paramsConfigMap.ToStringMap()} - } - return nil - } - - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &tstCfgSrc, - }) - - file := path.Join("testdata", "envvar_cfgsrc_mix.yaml") - cp, err := config.NewMapFromFile(file) - require.NoError(t, err) - - expectedFile := path.Join("testdata", "envvar_cfgsrc_mix_expected.yaml") - expectedConfigMap, err := config.NewMapFromFile(expectedFile) - require.NoError(t, err) - - res, err := manager.Resolve(ctx, cp) - require.NoError(t, err) - assert.Equal(t, expectedConfigMap.ToStringMap(), res.ToStringMap()) - assert.NoError(t, manager.Close(ctx)) -} - -func TestManager_parseStringValue(t *testing.T) { - ctx := context.Background() - manager := newManager(map[string]configsource.ConfigSource{ - "tstcfgsrc": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "str_key": {Value: "test_value"}, - "int_key": {Value: 1}, - "nil_key": {Value: nil}, - }, - }, - "tstcfgsrc/named": &testConfigSource{ - ValueMap: map[string]valueEntry{ - "int_key": {Value: 42}, - }, - }, - }) - - require.NoError(t, os.Setenv("envvar", "envvar_value")) - defer func() { - assert.NoError(t, os.Unsetenv("envvar")) - }() - require.NoError(t, os.Setenv("envvar_str_key", "str_key")) - defer func() { - assert.NoError(t, os.Unsetenv("envvar_str_key")) - }() - - tests := []struct { - want interface{} - wantErr error - name string - input string - }{ - { - name: "literal_string", - input: "literal_string", - want: "literal_string", - }, - { - name: "escaped_$", - input: "$$tstcfgsrc:int_key$$envvar", - want: "$tstcfgsrc:int_key$envvar", - }, - { - name: "cfgsrc_int", - input: "$tstcfgsrc:int_key", - want: 1, - }, - { - name: "concatenate_cfgsrc_string", - input: "prefix-$tstcfgsrc:str_key", - want: "prefix-test_value", - }, - { - name: "concatenate_cfgsrc_non_string", - input: "prefix-$tstcfgsrc:int_key", - want: "prefix-1", - }, - { - name: "envvar", - input: "$envvar", - want: "envvar_value", - }, - { - name: "prefixed_envvar", - input: "prefix-$envvar", - want: "prefix-envvar_value", - }, - { - name: "envvar_treated_as_cfgsrc", - input: "$envvar:suffix", - wantErr: &errUnknownConfigSource{}, - }, - { - name: "cfgsrc_using_envvar", - input: "$tstcfgsrc:$envvar_str_key", - want: "test_value", - }, - { - name: "envvar_cfgsrc_using_envvar", - input: "$envvar/$tstcfgsrc:$envvar_str_key", - want: "envvar_value/test_value", - }, - { - name: "delimited_cfgsrc", - input: "${tstcfgsrc:int_key}", - want: 1, - }, - { - name: "unknown_delimited_cfgsrc", - input: "${cfgsrc:int_key}", - wantErr: &errUnknownConfigSource{}, - }, - { - name: "delimited_cfgsrc_with_spaces", - input: "${ tstcfgsrc: int_key }", - want: 1, - }, - { - name: "interpolated_and_delimited_cfgsrc", - input: "0/${ tstcfgsrc: $envvar_str_key }/2/${tstcfgsrc:int_key}", - want: "0/test_value/2/1", - }, - { - name: "named_config_src", - input: "$tstcfgsrc/named:int_key", - want: 42, - }, - { - name: "named_config_src_bracketed", - input: "${tstcfgsrc/named:int_key}", - want: 42, - }, - { - name: "envvar_name_separator", - input: "$envvar/test/test", - want: "envvar_value/test/test", - }, - { - name: "envvar_treated_as_cfgsrc", - input: "$envvar/test:test", - wantErr: &errUnknownConfigSource{}, - }, - { - name: "retrieved_nil", - input: "${tstcfgsrc:nil_key}", - }, - { - name: "retrieved_nil_on_string", - input: "prefix-${tstcfgsrc:nil_key}-suffix", - want: "prefix--suffix", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := manager.parseStringValue(ctx, tt.input) - require.IsType(t, tt.wantErr, err) - require.Equal(t, tt.want, got) - }) - } -} - -func Test_parseCfgSrcInvocation(t *testing.T) { - tests := []struct { - params interface{} - name string - str string - cfgSrcName string - selector string - wantErr bool - }{ - { - name: "basic", - str: "cfgsrc:selector", - cfgSrcName: "cfgsrc", - selector: "selector", - }, - { - name: "missing_selector", - str: "cfgsrc", - wantErr: true, - }, - { - name: "params", - str: "cfgsrc:selector?p0=1&p1=a_string&p2=true", - cfgSrcName: "cfgsrc", - selector: "selector", - params: map[string]interface{}{ - "p0": 1, - "p1": "a_string", - "p2": true, - }, - }, - { - name: "query_pass_nil", - str: "cfgsrc:selector?p0&p1&p2", - cfgSrcName: "cfgsrc", - selector: "selector", - params: map[string]interface{}{ - "p0": nil, - "p1": nil, - "p2": nil, - }, - }, - { - name: "array_in_params", - str: "cfgsrc:selector?p0=0&p0=1&p0=2&p1=done", - cfgSrcName: "cfgsrc", - selector: "selector", - params: map[string]interface{}{ - "p0": []interface{}{0, 1, 2}, - "p1": "done", - }, - }, - { - name: "empty_param", - str: "cfgsrc:selector?no_closing=", - cfgSrcName: "cfgsrc", - selector: "selector", - params: map[string]interface{}{ - "no_closing": interface{}(nil), - }, - }, - { - name: "use_url_encode", - str: "cfgsrc:selector?p0=contains+%3D+and+%26+too", - cfgSrcName: "cfgsrc", - selector: "selector", - params: map[string]interface{}{ - "p0": "contains = and & too", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfgSrcName, selector, paramsConfigMap, err := parseCfgSrcInvocation(tt.str) - if tt.wantErr { - assert.Error(t, err) - return - } - - assert.NoError(t, err) - assert.Equal(t, tt.cfgSrcName, cfgSrcName) - assert.Equal(t, tt.selector, selector) - - paramsValue := (interface{})(nil) - if paramsConfigMap != nil { - paramsValue = paramsConfigMap.ToStringMap() - } - assert.Equal(t, tt.params, paramsValue) - }) - } -} - -func newManager(configSources map[string]configsource.ConfigSource) *Manager { - manager, _ := NewManager(nil) - manager.configSources = configSources - return manager -} - -// testConfigSource a ConfigSource to be used in tests. -type testConfigSource struct { - ValueMap map[string]valueEntry - - ErrOnRetrieve error - ErrOnClose error - - OnRetrieve func(ctx context.Context, selector string, paramsConfigMap *config.Map) error -} - -type valueEntry struct { - Value interface{} - WatchForUpdateFn func() error -} - -var _ configsource.ConfigSource = (*testConfigSource)(nil) - -func (t *testConfigSource) Retrieve(ctx context.Context, selector string, paramsConfigMap *config.Map) (configsource.Retrieved, error) { - if t.OnRetrieve != nil { - if err := t.OnRetrieve(ctx, selector, paramsConfigMap); err != nil { - return nil, err - } - } - - if t.ErrOnRetrieve != nil { - return nil, t.ErrOnRetrieve - } - - entry, ok := t.ValueMap[selector] - if !ok { - return nil, fmt.Errorf("no value for selector %q", selector) - } - - if entry.WatchForUpdateFn != nil { - return &watchableRetrieved{ - retrieved: retrieved{ - value: entry.Value, - }, - watchForUpdateFn: entry.WatchForUpdateFn, - }, nil - } - - return &retrieved{ - value: entry.Value, - }, nil -} - -func (t *testConfigSource) Close(context.Context) error { - return t.ErrOnClose -} - -type retrieved struct { - value interface{} -} - -var _ configsource.Retrieved = (*retrieved)(nil) - -func (r *retrieved) Value() interface{} { - return r.value -} - -type watchableRetrieved struct { - retrieved - watchForUpdateFn func() error -} - -func (r *watchableRetrieved) WatchForUpdate() error { - return r.watchForUpdateFn() -} From 1788c265bd1e52e372f63b4de248c6595665b921 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 18:34:33 -0500 Subject: [PATCH 21/23] Config sources to support full remote config and value substitution Resolves: https://github.com/open-telemetry/opentelemetry-collector/issues/4190 This is a draft PR (not production quality code) that shows roughly what the end goal is. Don't merge this PR, it is merely to show what the final destination looks like. If we agree that this is a good end goal I will split this into series of smaller PRs and will submit them. I am primarily looking for the feedback on the new concept and the APIs. There is no need to review all the implementation details, they are not production quality yet. ## Configuration File Format Changes Introduced 2 new optional sections in the configuration file format: config_sources and merge_configs. config_sources defines one or more configuration sources. Configuration sources are a new type of component that is pluggable like all other component types by implementing its Factory. Configuration sources must implement a configmapprovider.Provider interface. The merge_configs section contains a sequence of config source references. The configs from these sources is retrieved and merged in the specified order to form the top-level configuration of the Collector. The locally specified top-level configuration is merged last (thus has the highest precedence). This approach allows to maintain backward compatibility. Any previously valid configuration file remains valid. New configuration files may specify only config_sources/merge_configs section, move the local configuration to a separate file and use that file as one of the configuration sources. This approach also does not require us to introduce a new --config_sources command line option. ## Configuration Provider Interfaces Introduced the concept of config MapProvider and config ValueProvider, both implementing a base Provider interface. MapProvider is equivalent to the old Provider interface and allows retrieving configuration maps. ValueProvider can return configuration values, which includes primitive values, arrays, maps. ValueProvider also supports returning different values based on the selector and params passed which allows to parameterize the returned value at the place of usage. ## Command Line Invocation ValueProvicer-type config sources can be invoked directly from the command line. The --config option now accepts either a file path (which was the existing functionality) or values in the form of :? (which is a new functionality). The new functionality is triggered by the presence of a colon in the --config option value. This makes it impossible to pass file names that use a colon character, but we assume that this is an acceptable limitation since colon is often used as a file path delimiter character in Unix systems anyway. The new functionality allows for example for the following command lines: ./otelcol --config=config.yaml ./otelcol --config=files:config.yaml ./otelcol --config=http://example.com/path/to/config.yaml ./otelcol --config=http://example.com/path/to/config.yaml?foo=bar&bar=baz ## Example Config Sources As an example I implemented 3 config sources to demonstrate how things work: - env is a ValueProvider that allows getting environment variables by name. - files is both a MapProvider and a ValueProvider, which allows it to be used both as a top-level config source and as command line config source (see example above). - http is a ValueProvider which has factories that can be registered for "http" and "https" config sources and which can perform an HTTP GET operation to retrieve the config. ## Other Changes Deleted Expand map provider. It is no longer needed since environment expansion is now handled in the valueSubstitutor which needs to distinguish between env variables and config value source subsitution both starting with $ character. Deleted config/internal/configsource.Manager. The new functionality covers what Manager was intended to do. ## TODO - Refine RetrieveValue() to return a more specific type instead of interface{} - See if it is worth combining MapProvider and ValueProvider into just Provider (doesn't seem so, but need to check). - See if it possible/desirable to allow config source settings to be a single string such that `name` is not nessary for `files` config source and instead the file name can be specified directly as the setting of `files` source. - See if better names are possible for the new top-level config file keys. Perhaps root_configs is a better name instead of merge_configs. - Some more cleanup and refactoring may be necessary to tidy things up once we are fully settled on the functionality. Probably rename configmapprovider package to configprovider, rename Retrieve to RetriveMap, etc. - Add more comments to explain the design and how things work. - Add tests for everything. ## Example Usage From End User Perspective Below are some example usages. ### Example 1. Single Local Config File. This is the old (current) way. It is still fully supported and will continue to be supported. It is not obsolete and is the preferable way when there is a single local config file. Command line: `./otelcol --config=local.yaml` local.yaml content: ```yaml receivers: otlp: grpc: exporters: otlp: service: pipelines: traces: receivers: [otlp] exporters: [otlp] ``` ### Example 2. Config Sources, Single Local Config File. Command line: `./otelcol --config=sources.yaml` sources.yaml content: ```yaml config_sources: files: name: local.yaml merge_configs: [files] ``` local.yaml content: ```yaml receivers: otlp: grpc: exporters: otlp: service: pipelines: traces: receivers: [otlp] exporters: [otlp] ``` This example results in a config that is completely equivalent to the config in Example 1. ### Example 3. Config Sources from Command Line This uses a shorthand for specifying a single config source on the command line. Command line: `./otelcol --configs=files:local.yaml` local.yaml content: ```yaml receivers: otlp: grpc: exporters: otlp: service: pipelines: traces: receivers: [otlp] exporters: [otlp] ``` ### Example 4. Multiple Local Config Files Command line: `./otelcol --config=sources.yaml` sources.yaml content: ```yaml config_sources: # Merge all files that match the specified glob into one file and use that as a config files: name: /var/lib/otelcol/config/**/*.yaml merge_configs: [files] ``` ### Example 5. From HTTP Server Command line: `./otelcol --config=sources.yaml` source.yaml content: ```yaml config_sources: https: merge_configs: - https://example.com/path/file.yaml ``` This will do a HTTP GET request to https://example.com/path/file.yaml, will download the content and will use the content as the config file. The equivalent result can be achieved using only the command line: `./otelcol --config=https://example.com/path/file.yaml` ### Example 6. Multiple Sources Command line: `./otelcol --config=sources.yaml` source.yaml content: ```yaml config_sources: files: name: local.yaml s3: bucket: mybucket region: us-east-1 merge_configs: [files,s3] ``` This will merge a local.yaml file with the content of an S3 bucket and will use the content as the config file. ### Example 7. Value Sources Command line: `./otelcol --config=sources.yaml` source.yaml content: ```yaml config_sources: files: local.yaml # define a value source of "vault" type # that can be referenced later. vault: # endpoint is the Vault server address. endpoint: http://localhost:8200 # path is the Vault path to the secret location. path: secret/data/kv auth: token: foo merge_configs: [files] ``` local.yaml content: ```yaml receivers: otlp: grpc: exporters: signalfx: # this will read the access_token value from "vault" config source. realm: us0 access_token: $vault:data.access_token service: pipelines: metrics: receivers: [otlp] exporters: [signalfx] ``` ### Example 8. Environment Variables Command line: `./otelcol --config=config.yaml` config.yaml content: ```yaml config_sources: env: receivers: otlp: grpc: exporters: signalfx: # Both of the following values are read from env variables. realm: $SIGNALFX_REALM access_token: $env:SIGNALFX_ACCESSTOKEN service: pipelines: metrics: receivers: [otlp] exporters: [signalfx] ``` --- config/configmapprovider/valueprovider.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config/configmapprovider/valueprovider.go b/config/configmapprovider/valueprovider.go index ba788e20192..d130f58ee4b 100644 --- a/config/configmapprovider/valueprovider.go +++ b/config/configmapprovider/valueprovider.go @@ -62,6 +62,7 @@ type RetrievedValue interface { // should return immediately with ErrSessionClosed error. // Should never be called concurrently with itself. // If ctx is cancelled should return immediately with an error. + // TODO: use a more specific type for return value instead of interface{}. Get(ctx context.Context) (interface{}, error) // Close signals that the configuration for which it was used to retrieve values is From e8aeed2c6a12fba37ce59b8f82baa4341c63bf00 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 22:05:50 -0500 Subject: [PATCH 22/23] Improve ValueProvider interface comments. --- config/configmapprovider/valueprovider.go | 30 ++++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/config/configmapprovider/valueprovider.go b/config/configmapprovider/valueprovider.go index d130f58ee4b..2083f83bfbe 100644 --- a/config/configmapprovider/valueprovider.go +++ b/config/configmapprovider/valueprovider.go @@ -20,24 +20,35 @@ import ( "go.opentelemetry.io/collector/config" ) -// ValueProvider is an interface that helps to retrieve a config value and watch for any -// changes to the config value. Implementations may load the config from a file, -// a database or any other source. +// ValueProvider is an interface that helps to retrieve config values. Config values +// can be either a primitive value, such as string or number or a config.Map - a map +// of key/values. One ValueProvider may allow retrieving one or more values from the +// the same source (values are identified by the "selector", see below). +// ValueProvider allows to watch for any changes to the config values. +// Implementations may load the config value from a file, a database or any other source. type ValueProvider interface { Provider - // RetrieveValue goes to the configuration source and retrieves the selected data which - // contains the value to be injected in the configuration and the corresponding watcher that - // will be used to monitor for updates of the retrieved value. + // RetrieveValue goes to the configuration source and retrieves the selected data + // which contains the value to be injected in the configuration. // - // onChange callback is called when the config changes. onChange may be called from + // onChange callback is called when the config value changes. onChange may be called from // a different go routine. After onChange is called Retrieved.Get should be called - // to get the new config. See description of Retrieved for more details. + // to get the new config value. See description of RetrievedValue for more details. // onChange may be nil, which indicates that the caller is not interested in // knowing about the changes. // // If ctx is cancelled should return immediately with an error. // Should never be called concurrently with itself or with Shutdown. + // + // selector is an implementation-defined string which identifies which of the + // values that the provider has to retrieve. For example for a ValueProvider that + // retrieves values from process environment the selector may be the environment + // variable name. + // paramsConfigMap is any additional implementation-defined key/value map to + // pass to the ValueProvider to help retrieve the value. For example for a + // ValueProvider that retrieves values from an HTTP source the paramsConfigMap + // may contain HTTP URL query parameters. RetrieveValue(ctx context.Context, onChange func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) } @@ -57,7 +68,8 @@ type ValueProvider interface { // // ... // mapProvider.Shutdown() type RetrievedValue interface { - // Get returns the config Map. + // Get returns the config value, which should be either a primitive value, + // such as string or number or a *config.Map. // If Close is called before Get or concurrently with Get then Get // should return immediately with ErrSessionClosed error. // Should never be called concurrently with itself. From 50915c403b4db8aa0a43a7cff86f9ef481f1ac2c Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Mon, 22 Nov 2021 22:21:37 -0500 Subject: [PATCH 23/23] Fix linting --- config/configmapprovider/file.go | 12 +++--- config/configmapprovider/mock_test.go | 6 +-- config/configmapprovider/valueprovider.go | 2 +- config/configsource.go | 4 +- configsource/http/configsource.go | 9 +++- service/command.go | 1 + .../defaultconfigprovider.go | 42 +++++++++---------- .../defaultconfigprovider/valuesubstitutor.go | 3 +- 8 files changed, 41 insertions(+), 38 deletions(-) diff --git a/config/configmapprovider/file.go b/config/configmapprovider/file.go index ccc7f19c0b7..b359b2ef6e4 100644 --- a/config/configmapprovider/file.go +++ b/config/configmapprovider/file.go @@ -22,18 +22,18 @@ import ( "go.opentelemetry.io/collector/config" ) -type fileMapProvider struct { +type FileMapProvider struct { fileName string } // NewFile returns a new MapProvider that reads the configuration from the given file. -func NewFile(fileName string) *fileMapProvider { - return &fileMapProvider{ +func NewFile(fileName string) *FileMapProvider { + return &FileMapProvider{ fileName: fileName, } } -func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (RetrievedMap, error) { +func (fmp *FileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (RetrievedMap, error) { if fmp.fileName == "" { return nil, errors.New("config file not specified") } @@ -46,7 +46,7 @@ func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (R return &simpleRetrieved{confMap: cp}, nil } -func (fmp *fileMapProvider) RetrieveValue(_ context.Context, _ func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) { +func (fmp *FileMapProvider) RetrieveValue(_ context.Context, _ func(*ChangeEvent), selector string, paramsConfigMap *config.Map) (RetrievedValue, error) { if selector == "" { return nil, errors.New("config file not specified") } @@ -61,6 +61,6 @@ func (fmp *fileMapProvider) RetrieveValue(_ context.Context, _ func(*ChangeEvent return &simpleRetrievedValue{value: cp}, nil } -func (*fileMapProvider) Shutdown(context.Context) error { +func (*FileMapProvider) Shutdown(context.Context) error { return nil } diff --git a/config/configmapprovider/mock_test.go b/config/configmapprovider/mock_test.go index 66dd499b66a..d38bbb1ef2c 100644 --- a/config/configmapprovider/mock_test.go +++ b/config/configmapprovider/mock_test.go @@ -22,13 +22,13 @@ import ( // mockProvider is a mock implementation of Provider, useful for testing. type mockProvider struct { - retrieved Retrieved + retrieved RetrievedMap retrieveErr error } var _ Provider = &mockProvider{} -func (m *mockProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) { +func (m *mockProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (RetrievedMap, error) { return m.retrieved, m.retrieveErr } @@ -39,7 +39,7 @@ type mockRetrieved struct { getErr error } -var _ Retrieved = &mockRetrieved{} +var _ RetrievedMap = &mockRetrieved{} func (sr *mockRetrieved) Get(ctx context.Context) (*config.Map, error) { return sr.got, sr.getErr diff --git a/config/configmapprovider/valueprovider.go b/config/configmapprovider/valueprovider.go index 2083f83bfbe..73dd90e77e8 100644 --- a/config/configmapprovider/valueprovider.go +++ b/config/configmapprovider/valueprovider.go @@ -83,7 +83,7 @@ type RetrievedValue interface { // // This method must be called when the service ends, either in case of success or error. // - // Should never be called concurrently with itself. + // Should never be called concurrentdefaultconfigprovidery with itself. // May be called before, after or concurrently with Get. // If ctx is cancelled should return immediately with an error. // diff --git a/config/configsource.go b/config/configsource.go index 7e2b790cde8..70ffa1bbd20 100644 --- a/config/configsource.go +++ b/config/configsource.go @@ -17,7 +17,7 @@ package config // import "go.opentelemetry.io/collector/config" // ConfigSource is the configuration of a component.ConfigSource. // Specific ConfigSources must implement this interface and must embed // ConfigSourceSettings struct or a struct that extends it. -type ConfigSource interface { +type ConfigSource interface { //nolint identifiable validatable @@ -30,7 +30,7 @@ type ConfigSource interface { // It is highly recommended to "override" the Validate() function. // // When embedded in the ConfigSource config, it must be with `mapstructure:",squash"` tag. -type ConfigSourceSettings struct { +type ConfigSourceSettings struct { //nolint id ComponentID `mapstructure:"-"` } diff --git a/configsource/http/configsource.go b/configsource/http/configsource.go index bc631e40f56..cc230faadbf 100644 --- a/configsource/http/configsource.go +++ b/configsource/http/configsource.go @@ -43,7 +43,7 @@ func (r retrieved) Get(ctx context.Context) (interface{}, error) { for k, v := range r.paramsConfigMap.ToStringMap() { str, ok := v.(string) if !ok { - return nil, fmt.Errorf("invalid value for paramter %q", k) + return nil, fmt.Errorf("invalid value for parameter %q", k) } queryVals.Set(k, str) } @@ -54,7 +54,12 @@ func (r retrieved) Get(ctx context.Context) (interface{}, error) { } } - resp, err := http.Get(urlStr) + u, err := url.Parse(urlStr) + if err != nil { + return nil, fmt.Errorf("cannot load config from %s: %w", urlStr, err) + } + + resp, err := http.Get(u.String()) if err != nil { return nil, fmt.Errorf("cannot load config from %s: %w", urlStr, err) } diff --git a/service/command.go b/service/command.go index c92a93de403..25dfbe31c59 100644 --- a/service/command.go +++ b/service/command.go @@ -16,6 +16,7 @@ package service // import "go.opentelemetry.io/collector/service" import ( "github.com/spf13/cobra" + "go.opentelemetry.io/collector/service/defaultconfigprovider" ) diff --git a/service/defaultconfigprovider/defaultconfigprovider.go b/service/defaultconfigprovider/defaultconfigprovider.go index ab8b20cd1ab..ebf00316759 100644 --- a/service/defaultconfigprovider/defaultconfigprovider.go +++ b/service/defaultconfigprovider/defaultconfigprovider.go @@ -90,7 +90,8 @@ func (mp *defaultConfigProvider) Retrieve( // a provider for each. var rootProviders []configmapprovider.MapProvider for _, configSourceRef := range mergeConfigs { - mapProvider, err := mp.getConfigSourceMapProvider(configSourceRef, configSources) + var mapProvider configmapprovider.MapProvider + mapProvider, err = mp.getConfigSourceMapProvider(configSourceRef, configSources) if err != nil { return nil, err } @@ -143,22 +144,22 @@ func (mp *defaultConfigProvider) getConfigSourceMapProvider( configSourceName: cfgSrcName, } return mapProvider, nil + } - } else { - configSource, err := mp.findConfigSource(configSourceRef, configSources) - if err != nil { - return nil, err - } + configSource, err := mp.findConfigSource(configSourceRef, configSources) + if err != nil { + return nil, err + } - mapProvider, ok := configSource.(configmapprovider.MapProvider) - if !ok { - return nil, fmt.Errorf( - "config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", - configSource, - ) - } - return mapProvider, nil + mapProvider, ok := configSource.(configmapprovider.MapProvider) + if !ok { + return nil, fmt.Errorf( + "config source %q cannot be used in merge_configs section since it does not implement MapProvider interface", + configSource, + ) } + return mapProvider, nil + } func (mp *defaultConfigProvider) findConfigSource( @@ -204,29 +205,24 @@ func unmarshalSources(ctx context.Context, rootMap *config.Map, factories compon sourceType := id.Type() // See if we have a factory for this config source type. - factoryBase, ok := factories.ConfigSources[sourceType] + factory, ok := factories.ConfigSources[sourceType] if !ok { return nil, nil, fmt.Errorf("unknown config source type %q (did you register the config source factory?)", sourceType) } - cfg := factoryBase.CreateDefaultConfig() + cfg := factory.CreateDefaultConfig() cfg.SetIDName(sourceName) // Now that the default config struct is created we can Unmarshal into it, // and it will apply user-defined config on top of the default. - if err := configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { + if err = configunmarshaler.Unmarshal(config.NewMapFromStringMap(settings), cfg); err != nil { return nil, nil, fmt.Errorf("error reading config of config source %q: %w", sourceType, err) } - if err := cfg.Validate(); err != nil { + if err = cfg.Validate(); err != nil { return nil, nil, fmt.Errorf("config of config source %q is invalid: %w", sourceType, err) } - factory, ok := factoryBase.(component.ConfigSourceFactory) - if !ok { - return nil, nil, fmt.Errorf("config source %q does not implement ConfigSourceFactory", sourceType) - } - source, err := factory.CreateConfigSource(ctx, component.ConfigSourceCreateSettings{}, cfg) if err != nil { return nil, nil, err diff --git a/service/defaultconfigprovider/valuesubstitutor.go b/service/defaultconfigprovider/valuesubstitutor.go index a34d30ed270..51e86e90a07 100644 --- a/service/defaultconfigprovider/valuesubstitutor.go +++ b/service/defaultconfigprovider/valuesubstitutor.go @@ -9,9 +9,10 @@ import ( "strings" "github.com/spf13/cast" + "gopkg.in/yaml.v2" + "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/configmapprovider" - "gopkg.in/yaml.v2" ) const (