diff --git a/.gitignore b/.gitignore index 7c3fbd21c3535..b284a75bdae22 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ .DS_Store process.yml /.vscode +testdb.db diff --git a/accumulator.go b/accumulator.go index 1ea5737a84a99..95807fbe7a46c 100644 --- a/accumulator.go +++ b/accumulator.go @@ -52,6 +52,8 @@ type Accumulator interface { // Upgrade to a TrackingAccumulator with space for maxTracked // metrics/batches. WithTracking(maxTracked int) TrackingAccumulator + + WithNewMetricMaker(logName string, logger Logger, f func(metric Metric) Metric) Accumulator } // TrackingID uniquely identifies a tracked metric group diff --git a/agent/accumulator.go b/agent/accumulator.go index 3683b6767d47f..6c843ab9f47a9 100644 --- a/agent/accumulator.go +++ b/agent/accumulator.go @@ -1,6 +1,7 @@ package agent import ( + "sync" "time" "github.com/influxdata/telegraf" @@ -14,6 +15,8 @@ type MetricMaker interface { } type accumulator struct { + sync.Mutex + sync.Cond maker MetricMaker metrics chan<- telegraf.Metric precision time.Duration @@ -28,6 +31,8 @@ func NewAccumulator( metrics: metrics, precision: time.Nanosecond, } + acc.Cond.L = &acc.Mutex + return &acc } @@ -77,8 +82,13 @@ func (ac *accumulator) AddHistogram( } func (ac *accumulator) AddMetric(m telegraf.Metric) { + ac.Lock() + defer ac.Unlock() m.SetTime(m.Time().Round(ac.precision)) if m := ac.maker.MakeMetric(m); m != nil { + if ac.metrics == nil { + ac.Cond.Wait() // unlock and wait for metrics to be set. + } ac.metrics <- m } } @@ -91,11 +101,24 @@ func (ac *accumulator) addFields( t ...time.Time, ) { m := metric.New(measurement, tags, fields, ac.getTime(t), tp) + ac.Lock() + defer ac.Unlock() if m := ac.maker.MakeMetric(m); m != nil { + if ac.metrics == nil { + ac.Cond.Wait() // unlock and wait for metrics to be set. + } ac.metrics <- m } } +// setOutput changes the destination of the accumulator +func (ac *accumulator) setOutput(outCh chan<- telegraf.Metric) { + ac.Lock() + defer ac.Unlock() + ac.metrics = outCh + ac.Cond.Broadcast() +} + // AddError passes a runtime error to the accumulator. // The error will be tagged with the plugin name and written to the log. func (ac *accumulator) AddError(err error) { @@ -126,6 +149,32 @@ func (ac *accumulator) WithTracking(maxTracked int) telegraf.TrackingAccumulator } } +func (ac *accumulator) WithNewMetricMaker(logName string, logger telegraf.Logger, f func(m telegraf.Metric) telegraf.Metric) telegraf.Accumulator { + return &metricMakerAccumulator{ + Accumulator: ac, + makerFunc: f, + logName: logName, + logger: logger, + } +} + +type metricMakerAccumulator struct { + telegraf.Accumulator + logName string + logger telegraf.Logger + makerFunc func(m telegraf.Metric) telegraf.Metric +} + +func (ma *metricMakerAccumulator) LogName() string { + return ma.logName +} +func (ma *metricMakerAccumulator) Log() telegraf.Logger { + return ma.logger +} +func (ma *metricMakerAccumulator) MakeMetric(m telegraf.Metric) telegraf.Metric { + return ma.makerFunc(m) +} + type trackingAccumulator struct { telegraf.Accumulator delivered chan telegraf.DeliveryInfo diff --git a/agent/agent.go b/agent/agent.go index 78097bcd47731..92476da945911 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2,82 +2,97 @@ package agent import ( "context" + "errors" "fmt" "log" "os" + "reflect" "runtime" - "sort" "sync" "time" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/internal/channel" "github.com/influxdata/telegraf/models" - "github.com/influxdata/telegraf/plugins/serializers/influx" +) + +// agentState describes the running state of the agent. +// Plugins can only be ran once the agent is running, +// you can add plugins before the agent has started. +// you cannot remove plugins before the agent has started. +// you cannot add or remove plugins once it has started shutting down. +type agentState int8 + +const ( + agentStateStarting agentState = iota + agentStateRunning + agentStateShuttingDown ) // Agent runs a set of plugins. type Agent struct { Config *config.Config + + // units hold channels and define connections between plugins + outputGroupUnit outputGroupUnit + processorGroupUnit processorGroupUnit + // inputUnit inputUnit + inputGroupUnit inputGroupUnit + configPluginUnit configPluginUnit + + ctx context.Context + + stateChanged *sync.Cond // a condition that lets plugins know when when the agent state changes + state agentState } // NewAgent returns an Agent for the given Config. -func NewAgent(config *config.Config) (*Agent, error) { - a := &Agent{ - Config: config, +func NewAgent(ctx context.Context, cfg *config.Config) *Agent { + inputDestCh := make(chan telegraf.Metric) + outputSrcCh := make(chan telegraf.Metric) + + // by default, connect the dest of the inputs directly to the src for the outputs, + // as processors are added, they will be inserted between these two. + + return &Agent{ + Config: cfg, + ctx: ctx, + stateChanged: sync.NewCond(&sync.Mutex{}), + state: agentStateStarting, + inputGroupUnit: inputGroupUnit{ + dst: inputDestCh, + relay: channel.NewRelay(inputDestCh, outputSrcCh), + }, + outputGroupUnit: outputGroupUnit{ + src: outputSrcCh, + }, } - return a, nil } -// inputUnit is a group of input plugins and the shared channel they write to. -// -// ┌───────┐ -// │ Input │───┐ -// └───────┘ │ -// ┌───────┐ │ ______ -// │ Input │───┼──▶ ()_____) -// └───────┘ │ -// ┌───────┐ │ -// │ Input │───┘ -// └───────┘ type inputUnit struct { - dst chan<- telegraf.Metric - inputs []*models.RunningInput + input *models.RunningInput + cancelGather context.CancelFunc // used to cancel the gather loop for plugin shutdown +} + +type configPluginUnit struct { + sync.Mutex + plugins []config.ConfigPlugin } // ______ ┌───────────┐ ______ // ()_____)──▶ │ Processor │──▶ ()_____) // └───────────┘ type processorUnit struct { - src <-chan telegraf.Metric - dst chan<- telegraf.Metric - processor *models.RunningProcessor -} - -// aggregatorUnit is a group of Aggregators and their source and sink channels. -// Typically the aggregators write to a processor channel and pass the original -// metrics to the output channel. The sink channels may be the same channel. -// -// ┌────────────┐ -// ┌──▶ │ Aggregator │───┐ -// │ └────────────┘ │ -// ______ │ ┌────────────┐ │ ______ -// ()_____)───┼──▶ │ Aggregator │───┼──▶ ()_____) -// │ └────────────┘ │ -// │ ┌────────────┐ │ -// ├──▶ │ Aggregator │───┘ -// │ └────────────┘ -// │ ______ -// └────────────────────────▶ ()_____) -type aggregatorUnit struct { - src <-chan telegraf.Metric - aggC chan<- telegraf.Metric - outputC chan<- telegraf.Metric - aggregators []*models.RunningAggregator + order int + src chan telegraf.Metric // owns this src + dst chan<- telegraf.Metric // reference to another chan owned elsewhere + processor models.ProcessorRunner + accumulator *accumulator } -// outputUnit is a group of Outputs and their source channel. Metrics on the +// outputGroupUnit is a group of Outputs and their source channel. Metrics on the // channel are written to all outputs. // // ┌────────┐ @@ -85,350 +100,215 @@ type aggregatorUnit struct { // │ └────────┘ // ______ ┌─────┐ │ ┌────────┐ // ()_____)──▶ │ Fan │───┼──▶ │ Output │ -// └─────┘ │ └────────┘ +// src └─────┘ │ └────────┘ // │ ┌────────┐ // └──▶ │ Output │ // └────────┘ -type outputUnit struct { - src <-chan telegraf.Metric - outputs []*models.RunningOutput +type outputGroupUnit struct { + sync.Mutex + src chan telegraf.Metric + outputs []outputUnit } -// Run starts and runs the Agent until the context is done. -func (a *Agent) Run(ctx context.Context) error { - log.Printf("I! [agent] Config: Interval:%s, Quiet:%#v, Hostname:%#v, "+ - "Flush Interval:%s", - time.Duration(a.Config.Agent.Interval), a.Config.Agent.Quiet, - a.Config.Agent.Hostname, time.Duration(a.Config.Agent.FlushInterval)) - - log.Printf("D! [agent] Initializing plugins") - err := a.initPlugins() - if err != nil { - return err - } - - startTime := time.Now() +type outputUnit struct { + output *models.RunningOutput + cancelFlush context.CancelFunc // used to cancel the flush loop for plugin shutdown +} - log.Printf("D! [agent] Connecting outputs") - next, ou, err := a.startOutputs(ctx, a.Config.Outputs) - if err != nil { - return err - } +type processorGroupUnit struct { + sync.Mutex + processorUnits []*processorUnit +} - var apu []*processorUnit - var au *aggregatorUnit - if len(a.Config.Aggregators) != 0 { - aggC := next - if len(a.Config.AggProcessors) != 0 { - aggC, apu, err = a.startProcessors(next, a.Config.AggProcessors) - if err != nil { - return err - } +func (pg *processorGroupUnit) Find(pr models.ProcessorRunner) *processorUnit { + for _, unit := range pg.processorUnits { + if unit.processor.GetID() == pr.GetID() { + return unit } - - next, au = a.startAggregators(aggC, next, a.Config.Aggregators) } + return nil +} - var pu []*processorUnit - if len(a.Config.Processors) != 0 { - next, pu, err = a.startProcessors(next, a.Config.Processors) - if err != nil { - return err - } - } +// inputGroupUnit is a group of input plugins and the shared channel they write to. +// +// ┌───────┐ +// │ Input │───┐ +// └───────┘ │ +// ┌───────┐ │ ______ +// │ Input │───┼──▶ ()_____) +// └───────┘ │ +// ┌───────┐ │ +// │ Input │───┘ +// └───────┘ +type inputGroupUnit struct { + sync.Mutex + dst chan<- telegraf.Metric // owns channel; must stay open until app is shutting down + relay *channel.Relay + inputUnits []inputUnit +} + +// RunWithAPI runs Telegraf in API mode where all the plugins are controlled by +// the user through the config API. When running in this mode plugins are not +// loaded from the toml file. +// if ctx errors (eg cancels), inputs will be notified to shutdown +// during shutdown, once the output fanout has received the last metric, +// outputCancel is called as a sort of callback to notify the outputs to finish up. +// you don't want outputs subscribing to the same ctx as the inputs, +// otherwise they'd stop working before receiving all the messages (this is because +// they maintain their own internal buffers rather than depending on metrics buffered +// in a channel) +func (a *Agent) RunWithAPI(outputCancel context.CancelFunc) { + go func(outputCancel context.CancelFunc) { + a.runOutputFanout() + // then the fanout closes, notify the outputs that they won't be receiving + // more metrics via this cancel function. + outputCancel() // triggers outputs to finish up + }(outputCancel) - iu, err := a.startInputs(next, a.Config.Inputs) - if err != nil { - return err - } + log.Printf("I! [agent] Config: Interval:%s, Quiet:%#v, Hostname:%#v, "+ + "Flush Interval:%s", + time.Duration(a.Config.Agent.Interval), a.Config.Agent.Quiet, + a.Config.Agent.Hostname, time.Duration(a.Config.Agent.FlushInterval)) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - a.runOutputs(ou) - }() + a.inputGroupUnit.relay.Start() - if au != nil { - wg.Add(1) - go func() { - defer wg.Done() - a.runProcessors(apu) - }() - - wg.Add(1) - go func() { - defer wg.Done() - a.runAggregators(startTime, au) - }() - } + a.setState(agentStateRunning) - if pu != nil { - wg.Add(1) - go func() { - defer wg.Done() - a.runProcessors(pu) - }() - } + <-a.Context().Done() - wg.Add(1) - go func() { - defer wg.Done() - a.runInputs(ctx, startTime, iu) - }() + a.setState(agentStateShuttingDown) - wg.Wait() + // wait for all plugins to stop + a.waitForMainPluginsToStop() + a.waitForConfigPluginsToStop() log.Printf("D! [agent] Stopped Successfully") - return err } -// initPlugins runs the Init function on plugins. -func (a *Agent) initPlugins() error { - for _, input := range a.Config.Inputs { - err := input.Init() - if err != nil { - return fmt.Errorf("could not initialize input %s: %v", - input.LogName(), err) - } +// AddInput adds an input to the agent to be managed +func (a *Agent) AddInput(input *models.RunningInput) { + if a.isState(agentStateShuttingDown) { + return } - for _, processor := range a.Config.Processors { - err := processor.Init() - if err != nil { - return fmt.Errorf("could not initialize processor %s: %v", - processor.Config.Name, err) - } - } - for _, aggregator := range a.Config.Aggregators { - err := aggregator.Init() - if err != nil { - return fmt.Errorf("could not initialize aggregator %s: %v", - aggregator.Config.Name, err) - } - } - for _, processor := range a.Config.AggProcessors { - err := processor.Init() - if err != nil { - return fmt.Errorf("could not initialize processor %s: %v", - processor.Config.Name, err) - } - } - for _, output := range a.Config.Outputs { - err := output.Init() - if err != nil { - return fmt.Errorf("could not initialize output %s: %v", - output.Config.Name, err) - } - } - return nil + a.inputGroupUnit.Lock() + defer a.inputGroupUnit.Unlock() + + a.inputGroupUnit.inputUnits = append(a.inputGroupUnit.inputUnits, + inputUnit{ + input: input, + }) } -func (a *Agent) startInputs( - dst chan<- telegraf.Metric, - inputs []*models.RunningInput, -) (*inputUnit, error) { - log.Printf("D! [agent] Starting service inputs") +func (a *Agent) startInput(input *models.RunningInput) error { + // plugins can start before the agent has started; wait until it's asked to + // start before collecting metrics in case other plugins are still loading. + a.waitUntilState(agentStateRunning) - unit := &inputUnit{ - dst: dst, + a.inputGroupUnit.Lock() + // Service input plugins are not normally subject to timestamp + // rounding except for when precision is set on the input plugin. + // + // This only applies to the accumulator passed to Start(), the + // Gather() accumulator does apply rounding according to the + // precision and interval agent/plugin settings. + var interval time.Duration + var precision time.Duration + if input.Config.Precision != 0 { + precision = input.Config.Precision } - for _, input := range inputs { - if si, ok := input.Input.(telegraf.ServiceInput); ok { - // Service input plugins are not normally subject to timestamp - // rounding except for when precision is set on the input plugin. - // - // This only applies to the accumulator passed to Start(), the - // Gather() accumulator does apply rounding according to the - // precision and interval agent/plugin settings. - var interval time.Duration - var precision time.Duration - if input.Config.Precision != 0 { - precision = input.Config.Precision - } - - acc := NewAccumulator(input, dst) - acc.SetPrecision(getPrecision(precision, interval)) + // the plugin's Start() gets its own accumulator with no rounding, etc + acc := NewAccumulator(input, a.inputGroupUnit.dst) + acc.SetPrecision(getPrecision(precision, interval)) - err := si.Start(acc) - if err != nil { - stopServiceInputs(unit.inputs) - return nil, fmt.Errorf("starting input %s: %w", input.LogName(), err) - } + for _, inp := range a.inputGroupUnit.inputUnits { + if inp.input.GetID() == input.GetID() { + a.inputGroupUnit.Unlock() + return input.Start(acc) } - unit.inputs = append(unit.inputs, input) } + a.inputGroupUnit.Unlock() - return unit, nil + return errors.New("cannot start input; call AddInput first") } -// runInputs starts and triggers the periodic gather for Inputs. -// -// When the context is done the timers are stopped and this function returns -// after all ongoing Gather calls complete. -func (a *Agent) runInputs( - ctx context.Context, - startTime time.Time, - unit *inputUnit, -) { - var wg sync.WaitGroup - for _, input := range unit.inputs { - // Overwrite agent interval if this plugin has its own. - interval := time.Duration(a.Config.Agent.Interval) - if input.Config.Interval != 0 { - interval = input.Config.Interval - } - - // Overwrite agent precision if this plugin has its own. - precision := time.Duration(a.Config.Agent.Precision) - if input.Config.Precision != 0 { - precision = input.Config.Precision - } - - // Overwrite agent collection_jitter if this plugin has its own. - jitter := time.Duration(a.Config.Agent.CollectionJitter) - if input.Config.CollectionJitter != 0 { - jitter = input.Config.CollectionJitter - } - - var ticker Ticker - if a.Config.Agent.RoundInterval { - ticker = NewAlignedTicker(startTime, interval, jitter) - } else { - ticker = NewUnalignedTicker(interval, jitter) - } - defer ticker.Stop() - - acc := NewAccumulator(input, unit.dst) - acc.SetPrecision(getPrecision(precision, interval)) - - wg.Add(1) - go func(input *models.RunningInput) { - defer wg.Done() - a.gatherLoop(ctx, acc, input, ticker, interval) - }(input) +// RunInput is a blocking call that runs an input forever +func (a *Agent) RunInput(input *models.RunningInput, startTime time.Time) { + // default to agent interval but check for override + interval := time.Duration(a.Config.Agent.Interval) + if input.Config.Interval != 0 { + interval = input.Config.Interval + } + // default to agent precision but check for override + precision := time.Duration(a.Config.Agent.Precision) + if input.Config.Precision != 0 { + precision = input.Config.Precision + } + // default to agent collection_jitter but check for override + jitter := time.Duration(a.Config.Agent.CollectionJitter) + if input.Config.CollectionJitter != 0 { + jitter = input.Config.CollectionJitter } - wg.Wait() - - log.Printf("D! [agent] Stopping service inputs") - stopServiceInputs(unit.inputs) - - close(unit.dst) - log.Printf("D! [agent] Input channel closed") -} - -// testStartInputs is a variation of startInputs for use in --test and --once -// mode. It differs by logging Start errors and returning only plugins -// successfully started. -func (a *Agent) testStartInputs( - dst chan<- telegraf.Metric, - inputs []*models.RunningInput, -) *inputUnit { - log.Printf("D! [agent] Starting service inputs") - - unit := &inputUnit{ - dst: dst, + var ticker Ticker + if a.Config.Agent.RoundInterval { + ticker = NewAlignedTicker(startTime, interval, jitter) + } else { + ticker = NewUnalignedTicker(interval, jitter) } + defer ticker.Stop() - for _, input := range inputs { - if si, ok := input.Input.(telegraf.ServiceInput); ok { - // Service input plugins are not subject to timestamp rounding. - // This only applies to the accumulator passed to Start(), the - // Gather() accumulator does apply rounding according to the - // precision agent setting. - acc := NewAccumulator(input, dst) - acc.SetPrecision(time.Nanosecond) + acc := NewAccumulator(input, a.inputGroupUnit.dst) + acc.SetPrecision(getPrecision(precision, interval)) + ctx, cancelFunc := context.WithCancel(a.ctx) + defer cancelFunc() // just to keep linters happy - err := si.Start(acc) - if err != nil { - log.Printf("E! [agent] Starting input %s: %v", input.LogName(), err) - } + err := errors.New("loop at least once") + for err != nil && ctx.Err() == nil { + if err = a.startInput(input); err != nil { + log.Printf("E! [agent] failed to start plugin %q: %v", input.LogName(), err) + time.Sleep(10 * time.Second) } - - unit.inputs = append(unit.inputs, input) } - return unit -} - -// testRunInputs is a variation of runInputs for use in --test and --once mode. -// Instead of using a ticker to run the inputs they are called once immediately. -func (a *Agent) testRunInputs( - ctx context.Context, - wait time.Duration, - unit *inputUnit, -) { - var wg sync.WaitGroup - - nul := make(chan telegraf.Metric) - go func() { - for range nul { + a.inputGroupUnit.Lock() + for i, iu := range a.inputGroupUnit.inputUnits { + if iu.input == input { + a.inputGroupUnit.inputUnits[i].cancelGather = cancelFunc + break } - }() - - for _, input := range unit.inputs { - wg.Add(1) - go func(input *models.RunningInput) { - defer wg.Done() - - // Overwrite agent interval if this plugin has its own. - interval := time.Duration(a.Config.Agent.Interval) - if input.Config.Interval != 0 { - interval = input.Config.Interval - } - - // Overwrite agent precision if this plugin has its own. - precision := time.Duration(a.Config.Agent.Precision) - if input.Config.Precision != 0 { - precision = input.Config.Precision - } - - // Run plugins that require multiple gathers to calculate rate - // and delta metrics twice. - switch input.Config.Name { - case "cpu", "mongodb", "procstat": - nulAcc := NewAccumulator(input, nul) - nulAcc.SetPrecision(getPrecision(precision, interval)) - if err := input.Input.Gather(nulAcc); err != nil { - nulAcc.AddError(err) - } - - time.Sleep(500 * time.Millisecond) - } - - acc := NewAccumulator(input, unit.dst) - acc.SetPrecision(getPrecision(precision, interval)) - - if err := input.Input.Gather(acc); err != nil { - acc.AddError(err) - } - }(input) } - wg.Wait() + a.inputGroupUnit.Unlock() - internal.SleepContext(ctx, wait) + a.gatherLoop(ctx, acc, input, ticker, interval) + input.Stop() - log.Printf("D! [agent] Stopping service inputs") - stopServiceInputs(unit.inputs) - - close(unit.dst) - log.Printf("D! [agent] Input channel closed") -} - -// stopServiceInputs stops all service inputs. -func stopServiceInputs(inputs []*models.RunningInput) { - for _, input := range inputs { - if si, ok := input.Input.(telegraf.ServiceInput); ok { - si.Stop() + a.inputGroupUnit.Lock() + for i, iu := range a.inputGroupUnit.inputUnits { + if iu.input == input { + // remove from list + a.inputGroupUnit.inputUnits = append(a.inputGroupUnit.inputUnits[0:i], a.inputGroupUnit.inputUnits[i+1:]...) + break } } + a.inputGroupUnit.Unlock() } -// stopRunningOutputs stops all running outputs. -func stopRunningOutputs(outputs []*models.RunningOutput) { - for _, output := range outputs { - output.Close() +func (a *Agent) StopInput(i *models.RunningInput) { + defer a.inputGroupUnit.Unlock() +retry: + a.inputGroupUnit.Lock() + for _, iu := range a.inputGroupUnit.inputUnits { + if iu.input == i { + if iu.cancelGather == nil { + // plugin hasn't finished starting, wait longer. + a.inputGroupUnit.Unlock() + time.Sleep(1 * time.Millisecond) + goto retry + } + iu.cancelGather() + break + } } } @@ -489,144 +369,205 @@ func (a *Agent) gatherOnce( } } -// startProcessors sets up the processor chain and calls Start on all -// processors. If an error occurs any started processors are Stopped. -func (a *Agent) startProcessors( - dst chan<- telegraf.Metric, - processors models.RunningProcessors, -) (chan<- telegraf.Metric, []*processorUnit, error) { - var units []*processorUnit - - // Sort from last to first - sort.SliceStable(processors, func(i, j int) bool { - return processors[i].Config.Order > processors[j].Config.Order - }) - - var src chan telegraf.Metric - for _, processor := range processors { - src = make(chan telegraf.Metric, 100) - acc := NewAccumulator(processor, dst) - - err := processor.Start(acc) - if err != nil { - for _, u := range units { - u.processor.Stop() - close(u.dst) +func (a *Agent) AddProcessor(processor models.ProcessorRunner) { + if a.isState(agentStateShuttingDown) { + return + } + a.inputGroupUnit.Lock() + defer a.inputGroupUnit.Unlock() + a.processorGroupUnit.Lock() + defer a.processorGroupUnit.Unlock() + + pu := processorUnit{ + src: make(chan telegraf.Metric, 100), + processor: processor, + } + + // insertPos is the position in the list after the slice insert operation + insertPos := 0 + + if len(a.processorGroupUnit.processorUnits) > 0 { + // figure out where in the list to put us + for i, unit := range a.processorGroupUnit.processorUnits { + if unit.order > int(processor.Order()) { + break } - return nil, nil, fmt.Errorf("starting processor %s: %w", processor.LogName(), err) + insertPos = i + 1 } + } - units = append(units, &processorUnit{ - src: src, - dst: dst, - processor: processor, - }) + // if we're the last processor in the list + if insertPos == len(a.processorGroupUnit.processorUnits) { + pu.dst = a.outputGroupUnit.src + } else { + // we're not the last processor + pu.dst = a.processorGroupUnit.processorUnits[insertPos].src + } + + acc := NewAccumulator(processor.(MetricMaker), pu.dst) + pu.accumulator = acc.(*accumulator) - dst = src + // only if we're the first processor or being inserted at front-of-line: + if insertPos == 0 { + a.inputGroupUnit.relay.SetDest(pu.src) + } else { + // not the first processor to be added + prev := a.processorGroupUnit.processorUnits[insertPos-1] + prev.accumulator.setOutput(pu.src) } - return src, units, nil + list := a.processorGroupUnit.processorUnits + // list[0:insertPos] + pu + list[insertPos:] + // list = [0,1,2,3,4] + // [0,1] + pu + [2,3,4] + a.processorGroupUnit.processorUnits = append(append(list[0:insertPos], &pu), list[insertPos:]...) } -// runProcessors begins processing metrics and runs until the source channel is -// closed and all metrics have been written. -func (a *Agent) runProcessors( - units []*processorUnit, -) { - var wg sync.WaitGroup - for _, unit := range units { - wg.Add(1) - go func(unit *processorUnit) { - defer wg.Done() - - acc := NewAccumulator(unit.processor, unit.dst) - for m := range unit.src { - if err := unit.processor.Add(m, acc); err != nil { - acc.AddError(err) - m.Drop() - } +func (a *Agent) startProcessor(processor models.ProcessorRunner) error { + a.processorGroupUnit.Lock() + + for _, pu := range a.processorGroupUnit.processorUnits { + if pu.processor.GetID() == processor.GetID() { + a.processorGroupUnit.Unlock() + err := processor.Start(pu.accumulator) + if err != nil { + return fmt.Errorf("starting processor %s: %w", processor.LogName(), err) } - unit.processor.Stop() - close(unit.dst) - log.Printf("D! [agent] Processor channel closed") - }(unit) + + return nil + } } - wg.Wait() + + a.processorGroupUnit.Unlock() + return nil } -// startAggregators sets up the aggregator unit and returns the source channel. -func (a *Agent) startAggregators( - aggC chan<- telegraf.Metric, - outputC chan<- telegraf.Metric, - aggregators []*models.RunningAggregator, -) (chan<- telegraf.Metric, *aggregatorUnit) { - src := make(chan telegraf.Metric, 100) - unit := &aggregatorUnit{ - src: src, - aggC: aggC, - outputC: outputC, - aggregators: aggregators, +// RunProcessor is a blocking call that runs a processor forever +func (a *Agent) RunProcessor(p models.ProcessorRunner) { + a.processorGroupUnit.Lock() + pu := a.processorGroupUnit.Find(p) + a.processorGroupUnit.Unlock() + + if pu == nil { + log.Print("W! [agent] Must call AddProcessor before calling RunProcessor") + return } - return src, unit -} -// runAggregators beings aggregating metrics and runs until the source channel -// is closed and all metrics have been written. -func (a *Agent) runAggregators( - startTime time.Time, - unit *aggregatorUnit, -) { - ctx, cancel := context.WithCancel(context.Background()) + err := errors.New("loop at least once") + for err != nil && a.ctx.Err() == nil { + if err = a.startProcessor(p); err != nil { + log.Printf("E! [agent] failed to start processor %q: %v", p.LogName(), err) + time.Sleep(10 * time.Second) + } + } - // Before calling Add, initialize the aggregation window. This ensures - // that any metric created after start time will be aggregated. - for _, agg := range a.Config.Aggregators { - since, until := updateWindow(startTime, a.Config.Agent.RoundInterval, agg.Period()) - agg.UpdateWindow(since, until) + // processors own their src channel, if it's closed either it's been asked to stop or we're shutting down + for m := range pu.src { + if err := pu.processor.Add(m, pu.accumulator); err != nil { + pu.accumulator.AddError(err) + m.Reject() + } } - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - for metric := range unit.src { - var dropOriginal bool - for _, agg := range a.Config.Aggregators { - if ok := agg.Add(metric); ok { - dropOriginal = true - } - } + pu.processor.Stop() + // only close dst channel if we're shutting down + if a.ctx.Err() != nil { + a.processorGroupUnit.Lock() + if pu.dst != nil { + close(pu.dst) + } + a.processorGroupUnit.Unlock() + } + + a.processorGroupUnit.Lock() + defer a.processorGroupUnit.Unlock() + for i, curr := range a.processorGroupUnit.processorUnits { + if pu == curr { + // remove it from the slice + // Remove the stopped processor from the units list + a.processorGroupUnit.processorUnits = append(a.processorGroupUnit.processorUnits[0:i], a.processorGroupUnit.processorUnits[i+1:len(a.processorGroupUnit.processorUnits)]...) + } + } +} + +// StopProcessor stops processors or aggregators. +// ProcessorRunner could be a *models.RunningProcessor or a *models.RunningAggregator +func (a *Agent) StopProcessor(p models.ProcessorRunner) { + if a.isState(agentStateShuttingDown) { + return + } - if !dropOriginal { - unit.outputC <- metric // keep original. + a.processorGroupUnit.Lock() + defer a.processorGroupUnit.Unlock() + + for i, curr := range a.processorGroupUnit.processorUnits { + if p.GetID() == curr.processor.GetID() { + if i == 0 { + a.inputGroupUnit.relay.SetDest(curr.dst) } else { - metric.Drop() + prev := a.processorGroupUnit.processorUnits[i-1] + prev.accumulator.setOutput(curr.dst) } + close(curr.src) // closing source will tell the processor to stop. } - cancel() - }() + } +} + +// RunProcessor is a blocking call that runs a processor forever +func (a *Agent) RunConfigPlugin(ctx context.Context, plugin config.ConfigPlugin) { + a.configPluginUnit.Lock() + a.configPluginUnit.plugins = append(a.configPluginUnit.plugins, plugin) + a.configPluginUnit.Unlock() - for _, agg := range a.Config.Aggregators { - wg.Add(1) - go func(agg *models.RunningAggregator) { - defer wg.Done() + <-ctx.Done() + a.waitForMainPluginsToStop() - interval := time.Duration(a.Config.Agent.Interval) - precision := time.Duration(a.Config.Agent.Precision) + if err := plugin.Close(); err != nil { + log.Printf("E! [agent] Configuration plugin failed to close: %v", err) + } + + // because we don't have wrappers for config plugins, check to see if there's a storage plugin attached and close it. + p := reflect.ValueOf(plugin) + if p.Kind() == reflect.Ptr { + p = p.Elem() + } + if v := p.FieldByName("Storage"); !v.IsZero() && v.IsValid() && !v.IsNil() { + if sp, ok := v.Interface().(config.StoragePlugin); ok { + if err := sp.Close(); err != nil { + log.Printf("E! [agent] Storage plugin failed to close: %v", err) + } + } + } - acc := NewAccumulator(agg, unit.aggC) - acc.SetPrecision(getPrecision(precision, interval)) - a.push(ctx, agg, acc) - }(agg) + a.configPluginUnit.Lock() + pos := -1 + for i, p := range a.configPluginUnit.plugins { + if p == plugin { + pos = i + break + } + } + if pos > -1 { + a.configPluginUnit.plugins = append(a.configPluginUnit.plugins[0:pos], a.configPluginUnit.plugins[pos+1:]...) } + a.configPluginUnit.Unlock() +} - wg.Wait() +func (a *Agent) StopOutput(output *models.RunningOutput) { + if a.isState(agentStateShuttingDown) { + return + } + a.outputGroupUnit.Lock() + defer a.outputGroupUnit.Unlock() - // In the case that there are no processors, both aggC and outputC are the - // same channel. If there are processors, we close the aggC and the - // processor chain will close the outputC when it finishes processing. - close(unit.aggC) - log.Printf("D! [agent] Aggregator channel closed") + // find plugin + for _, o := range a.outputGroupUnit.outputs { + if o.output == output { + if o.cancelFlush != nil { + o.cancelFlush() + } + } + } } func updateWindow(start time.Time, roundInterval bool, period time.Duration) (time.Time, time.Time) { @@ -645,57 +586,27 @@ func updateWindow(start time.Time, roundInterval bool, period time.Duration) (ti return since, until } -// push runs the push for a single aggregator every period. -func (a *Agent) push( - ctx context.Context, - aggregator *models.RunningAggregator, - acc telegraf.Accumulator, -) { - for { - // Ensures that Push will be called for each period, even if it has - // already elapsed before this function is called. This is guaranteed - // because so long as only Push updates the EndPeriod. This method - // also avoids drift by not using a ticker. - until := time.Until(aggregator.EndPeriod()) - - select { - case <-time.After(until): - aggregator.Push(acc) - break - case <-ctx.Done(): - aggregator.Push(acc) - return - } +func (a *Agent) AddOutput(output *models.RunningOutput) { + if a.isState(agentStateShuttingDown) { + return } + a.outputGroupUnit.Lock() + a.outputGroupUnit.outputs = append(a.outputGroupUnit.outputs, outputUnit{output: output}) + a.outputGroupUnit.Unlock() } -// startOutputs calls Connect on all outputs and returns the source channel. -// If an error occurs calling Connect all stared plugins have Close called. -func (a *Agent) startOutputs( - ctx context.Context, - outputs []*models.RunningOutput, -) (chan<- telegraf.Metric, *outputUnit, error) { - src := make(chan telegraf.Metric, 100) - unit := &outputUnit{src: src} - for _, output := range outputs { - err := a.connectOutput(ctx, output) - if err != nil { - for _, output := range unit.outputs { - output.Close() - } - return nil, nil, fmt.Errorf("connecting output %s: %w", output.LogName(), err) - } - - unit.outputs = append(unit.outputs, output) +func (a *Agent) startOutput(output *models.RunningOutput) error { + if err := a.connectOutput(a.ctx, output); err != nil { + return fmt.Errorf("connecting output %s: %w", output.LogName(), err) } - return src, unit, nil + return nil } // connectOutputs connects to all outputs. func (a *Agent) connectOutput(ctx context.Context, output *models.RunningOutput) error { log.Printf("D! [agent] Attempting connection to [%s]", output.LogName()) - err := output.Output.Connect() + err := output.Connect() if err != nil { log.Printf("E! [agent] Failed to connect to [%s], retrying in 15s, "+ "error was '%s'", output.LogName(), err) @@ -705,7 +616,7 @@ func (a *Agent) connectOutput(ctx context.Context, output *models.RunningOutput) return err } - err = output.Output.Connect() + err = output.Connect() if err != nil { return fmt.Errorf("Error connecting to output %q: %w", output.LogName(), err) } @@ -714,64 +625,84 @@ func (a *Agent) connectOutput(ctx context.Context, output *models.RunningOutput) return nil } -// runOutputs begins processing metrics and returns until the source channel is -// closed and all metrics have been written. On shutdown metrics will be -// written one last time and dropped if unsuccessful. -func (a *Agent) runOutputs( - unit *outputUnit, -) { - var wg sync.WaitGroup +// RunOutput runs an output; note the context should be a special context that +// only cancels when it's time for the outputs to close: when the main context +// has closed AND all the input and processor plugins are done. +func (a *Agent) RunOutput(outputCtx context.Context, output *models.RunningOutput) { + // wrap with a cancel context so that the StopOutput can stop this individual output without stopping all the outputs. + ctx, cancel := context.WithCancel(outputCtx) + defer cancel() + + a.outputGroupUnit.Lock() + for i, o := range a.outputGroupUnit.outputs { + if o.output == output { + a.outputGroupUnit.outputs[i].cancelFlush = cancel + } + } + a.outputGroupUnit.Unlock() - // Start flush loop interval := time.Duration(a.Config.Agent.FlushInterval) jitter := time.Duration(a.Config.Agent.FlushJitter) + // Overwrite agent flush_interval if this plugin has its own. + if output.Config.FlushInterval != 0 { + interval = output.Config.FlushInterval + } - ctx, cancel := context.WithCancel(context.Background()) + // Overwrite agent flush_jitter if this plugin has its own. + if output.Config.FlushJitter != 0 { + jitter = output.Config.FlushJitter + } - for _, output := range unit.outputs { - interval := interval - // Overwrite agent flush_interval if this plugin has its own. - if output.Config.FlushInterval != 0 { - interval = output.Config.FlushInterval - } + ticker := NewRollingTicker(interval, jitter) + defer ticker.Stop() - jitter := jitter - // Overwrite agent flush_jitter if this plugin has its own. - if output.Config.FlushJitter != 0 { - jitter = output.Config.FlushJitter + err := errors.New("loop at least once") + for err != nil && a.ctx.Err() == nil { + if err = a.startOutput(output); err != nil { + log.Printf("E! [agent] failed to start output %q: %v", output.LogName(), err) + time.Sleep(10 * time.Second) } + } - wg.Add(1) - go func(output *models.RunningOutput) { - defer wg.Done() + a.flushLoop(ctx, output, ticker) - ticker := NewRollingTicker(interval, jitter) - defer ticker.Stop() + a.outputGroupUnit.Lock() - a.flushLoop(ctx, output, ticker) - }(output) + // find plugin + for i, o := range a.outputGroupUnit.outputs { + if o.output == output { + // disconnect it from the output broadcaster and remove it from the list + a.outputGroupUnit.outputs = append(a.outputGroupUnit.outputs[:i], a.outputGroupUnit.outputs[i+1:]...) + } + } + a.outputGroupUnit.Unlock() + + if err = a.flushOnce(output, ticker, output.Write); err != nil { + log.Printf("E! [agent] Error writing to %s: %v", output.LogName(), err) } - for metric := range unit.src { - for i, output := range unit.outputs { - if i == len(a.Config.Outputs)-1 { - output.AddMetric(metric) + output.Close() +} + +// runOutputFanout does the outputGroupUnit fanout, copying a metric to all outputs +func (a *Agent) runOutputFanout() { + for metric := range a.outputGroupUnit.src { + // if there are no outputs I guess we're dropping them. + a.outputGroupUnit.Lock() + outs := a.outputGroupUnit.outputs + a.outputGroupUnit.Unlock() + for i, output := range outs { + if i == len(outs)-1 { + output.output.AddMetric(metric) } else { - output.AddMetric(metric.Copy()) + output.output.AddMetric(metric.Copy()) } } } - - log.Println("I! [agent] Hang on, flushing any cached metrics before shutdown") - cancel() - wg.Wait() - - log.Println("I! [agent] Stopping running outputs") - stopRunningOutputs(unit.outputs) + // closing of outputs is done by RunOutput } -// flushLoop runs an output's flush function periodically until the context is -// done. +// flushLoop runs an output's flush function periodically until the context is done. func (a *Agent) flushLoop( ctx context.Context, output *models.RunningOutput, @@ -790,11 +721,9 @@ func (a *Agent) flushLoop( for { // Favor shutdown over other methods. - select { - case <-ctx.Done(): + if ctx.Err() != nil { logError(a.flushOnce(output, ticker, output.Write)) return - default: } select { @@ -838,246 +767,154 @@ func (a *Agent) flushOnce( } } -// Test runs the inputs, processors and aggregators for a single gather and -// writes the metrics to stdout. -func (a *Agent) Test(ctx context.Context, wait time.Duration) error { - src := make(chan telegraf.Metric, 100) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - s := influx.NewSerializer() - s.SetFieldSortOrder(influx.SortFields) - - for metric := range src { - octets, err := s.Serialize(metric) - if err == nil { - fmt.Print("> ", string(octets)) - } - metric.Reject() - } - }() - - err := a.test(ctx, wait, src) - if err != nil { - return err +// Returns the rounding precision for metrics. +func getPrecision(precision, interval time.Duration) time.Duration { + if precision > 0 { + return precision } - wg.Wait() - - if models.GlobalGatherErrors.Get() != 0 { - return fmt.Errorf("input plugins recorded %d errors", models.GlobalGatherErrors.Get()) + switch { + case interval >= time.Second: + return time.Second + case interval >= time.Millisecond: + return time.Millisecond + case interval >= time.Microsecond: + return time.Microsecond + default: + return time.Nanosecond } - return nil } -// Test runs the agent and performs a single gather sending output to the -// outputF. After gathering pauses for the wait duration to allow service -// inputs to run. -func (a *Agent) test(ctx context.Context, wait time.Duration, outputC chan<- telegraf.Metric) error { - log.Printf("D! [agent] Initializing plugins") - err := a.initPlugins() - if err != nil { - return err - } - - startTime := time.Now() - - next := outputC - - var apu []*processorUnit - var au *aggregatorUnit - if len(a.Config.Aggregators) != 0 { - procC := next - if len(a.Config.AggProcessors) != 0 { - procC, apu, err = a.startProcessors(next, a.Config.AggProcessors) - if err != nil { - return err - } - } - - next, au = a.startAggregators(procC, next, a.Config.Aggregators) +// panicRecover displays an error if an input panics. +func panicRecover(input *models.RunningInput) { + // nolint:revive // this function must be called with defer + if err := recover(); err != nil { + trace := make([]byte, 2048) + runtime.Stack(trace, true) + log.Printf("E! FATAL: [%s] panicked: %s, Stack:\n%s", + input.LogName(), err, trace) + log.Println("E! PLEASE REPORT THIS PANIC ON GITHUB with " + + "stack trace, configuration, and OS information: " + + "https://github.com/influxdata/telegraf/issues/new/choose") } +} - var pu []*processorUnit - if len(a.Config.Processors) != 0 { - next, pu, err = a.startProcessors(next, a.Config.Processors) - if err != nil { - return err - } - } +func (a *Agent) RunningInputs() []*models.RunningInput { + a.inputGroupUnit.Lock() + defer a.inputGroupUnit.Unlock() + runningInputs := []*models.RunningInput{} - iu := a.testStartInputs(next, a.Config.Inputs) - - var wg sync.WaitGroup - if au != nil { - wg.Add(1) - go func() { - defer wg.Done() - a.runProcessors(apu) - }() - - wg.Add(1) - go func() { - defer wg.Done() - a.runAggregators(startTime, au) - }() + for _, iu := range a.inputGroupUnit.inputUnits { + runningInputs = append(runningInputs, iu.input) } + return runningInputs +} - if pu != nil { - wg.Add(1) - go func() { - defer wg.Done() - a.runProcessors(pu) - }() +func (a *Agent) RunningProcessors() []models.ProcessorRunner { + a.processorGroupUnit.Lock() + defer a.processorGroupUnit.Unlock() + runningProcessors := []models.ProcessorRunner{} + for _, pu := range a.processorGroupUnit.processorUnits { + runningProcessors = append(runningProcessors, pu.processor) } - - wg.Add(1) - go func() { - defer wg.Done() - a.testRunInputs(ctx, wait, iu) - }() - - wg.Wait() - - log.Printf("D! [agent] Stopped Successfully") - - return nil + return runningProcessors } -// Once runs the full agent for a single gather. -func (a *Agent) Once(ctx context.Context, wait time.Duration) error { - err := a.once(ctx, wait) - if err != nil { - return err +func (a *Agent) RunningOutputs() []*models.RunningOutput { + a.outputGroupUnit.Lock() + defer a.outputGroupUnit.Unlock() + runningOutputs := []*models.RunningOutput{} + // make sure we allocate and use a new slice that doesn't need a lock + for _, o := range a.outputGroupUnit.outputs { + runningOutputs = append(runningOutputs, o.output) } - if models.GlobalGatherErrors.Get() != 0 { - return fmt.Errorf("input plugins recorded %d errors", models.GlobalGatherErrors.Get()) - } + return runningOutputs +} - unsent := 0 - for _, output := range a.Config.Outputs { - unsent += output.BufferLength() - } - if unsent != 0 { - return fmt.Errorf("output plugins unable to send %d metrics", unsent) - } - return nil +func (a *Agent) Context() context.Context { + a.inputGroupUnit.Lock() + defer a.inputGroupUnit.Unlock() + return a.ctx } -// On runs the agent and performs a single gather sending output to the -// outputF. After gathering pauses for the wait duration to allow service -// inputs to run. -func (a *Agent) once(ctx context.Context, wait time.Duration) error { - log.Printf("D! [agent] Initializing plugins") - err := a.initPlugins() - if err != nil { - return err +// waitForMainPluginsToStop waits for inputs, processors, and outputs to stop. +func (a *Agent) waitForMainPluginsToStop() { + for { + a.inputGroupUnit.Lock() + if len(a.inputGroupUnit.inputUnits) > 0 { + // fmt.Printf("waiting for %d inputs\n", len(a.inputGroupUnit.inputUnits)) + a.inputGroupUnit.Unlock() + time.Sleep(100 * time.Millisecond) + continue + } + break } - startTime := time.Now() - - log.Printf("D! [agent] Connecting outputs") - next, ou, err := a.startOutputs(ctx, a.Config.Outputs) - if err != nil { - return err + if a.inputGroupUnit.dst != nil { + close(a.inputGroupUnit.dst) + a.inputGroupUnit.dst = nil } - - var apu []*processorUnit - var au *aggregatorUnit - if len(a.Config.Aggregators) != 0 { - procC := next - if len(a.Config.AggProcessors) != 0 { - procC, apu, err = a.startProcessors(next, a.Config.AggProcessors) - if err != nil { - return err - } + a.inputGroupUnit.Unlock() + for { + time.Sleep(100 * time.Millisecond) + a.processorGroupUnit.Lock() + if len(a.processorGroupUnit.processorUnits) > 0 { + // fmt.Printf("waiting for %d processors\n", len(a.processorGroupUnit.processorUnits)) + a.processorGroupUnit.Unlock() + time.Sleep(100 * time.Millisecond) + continue } - - next, au = a.startAggregators(procC, next, a.Config.Aggregators) + break } - - var pu []*processorUnit - if len(a.Config.Processors) != 0 { - next, pu, err = a.startProcessors(next, a.Config.Processors) - if err != nil { - return err + a.processorGroupUnit.Unlock() + for { + a.outputGroupUnit.Lock() + if len(a.outputGroupUnit.outputs) > 0 { + a.outputGroupUnit.Unlock() + time.Sleep(100 * time.Millisecond) + continue } + break } + a.outputGroupUnit.Unlock() +} - iu := a.testStartInputs(next, a.Config.Inputs) - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - a.runOutputs(ou) - }() - - if au != nil { - wg.Add(1) - go func() { - defer wg.Done() - a.runProcessors(apu) - }() - - wg.Add(1) - go func() { - defer wg.Done() - a.runAggregators(startTime, au) - }() - } - - if pu != nil { - wg.Add(1) - go func() { - defer wg.Done() - a.runProcessors(pu) - }() +// waitForConfigPluginsToStop waits for config plugins to stop +func (a *Agent) waitForConfigPluginsToStop() { + for { + a.configPluginUnit.Lock() + if len(a.configPluginUnit.plugins) > 0 { + // fmt.Printf("waiting for %d config plugins\n", len(a.configPluginUnit.plugins)) + a.configPluginUnit.Unlock() + continue + } + break } - - wg.Add(1) - go func() { - defer wg.Done() - a.testRunInputs(ctx, wait, iu) - }() - - wg.Wait() - - log.Printf("D! [agent] Stopped Successfully") - - return nil + a.configPluginUnit.Unlock() + // everything closed; shut down } -// Returns the rounding precision for metrics. -func getPrecision(precision, interval time.Duration) time.Duration { - if precision > 0 { - return precision - } +// setState sets the agent's internal state. +func (a *Agent) setState(newState agentState) { + a.stateChanged.L.Lock() + a.state = newState + a.stateChanged.Broadcast() + a.stateChanged.L.Unlock() +} - switch { - case interval >= time.Second: - return time.Second - case interval >= time.Millisecond: - return time.Millisecond - case interval >= time.Microsecond: - return time.Microsecond - default: - return time.Nanosecond - } +// isState returns true if the agent state matches the state parameter +func (a *Agent) isState(state agentState) bool { + a.stateChanged.L.Lock() + defer a.stateChanged.L.Unlock() + return a.state == state } -// panicRecover displays an error if an input panics. -func panicRecover(input *models.RunningInput) { - if err := recover(); err != nil { - trace := make([]byte, 2048) - runtime.Stack(trace, true) - log.Printf("E! FATAL: [%s] panicked: %s, Stack:\n%s", - input.LogName(), err, trace) - log.Println("E! PLEASE REPORT THIS PANIC ON GITHUB with " + - "stack trace, configuration, and OS information: " + - "https://github.com/influxdata/telegraf/issues/new/choose") +// waitUntilState waits until the agent state is the requested state. +func (a *Agent) waitUntilState(state agentState) { + a.stateChanged.L.Lock() + for a.state != state { + a.stateChanged.Wait() } + a.stateChanged.L.Unlock() } diff --git a/agent/agent_test.go b/agent/agent_test.go index 9cc631b17c465..7aa3a118d76de 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -1,111 +1,92 @@ package agent import ( + "context" + "strings" "testing" "time" "github.com/influxdata/telegraf/config" _ "github.com/influxdata/telegraf/plugins/inputs/all" _ "github.com/influxdata/telegraf/plugins/outputs/all" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestAgent_OmitHostname(t *testing.T) { c := config.NewConfig() c.Agent.OmitHostname = true - _, err := NewAgent(c) - assert.NoError(t, err) - assert.NotContains(t, c.Tags, "host") + require.NotContains(t, c.Tags, "host") } func TestAgent_LoadPlugin(t *testing.T) { c := config.NewConfig() c.InputFilters = []string{"mysql"} - err := c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ := NewAgent(c) - assert.Equal(t, 1, len(a.Config.Inputs)) + a := NewAgent(context.Background(), c) + c.SetAgent(a) + err := c.LoadConfig(context.Background(), context.Background(), "../config/testdata/telegraf-agent.toml") + require.NoError(t, err) + require.Equal(t, 1, len(a.Config.Inputs())) c = config.NewConfig() c.InputFilters = []string{"foo"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 0, len(a.Config.Inputs)) + a = NewAgent(context.Background(), c) + c.SetAgent(a) + err = c.LoadConfig(context.Background(), context.Background(), "../config/testdata/telegraf-agent.toml") + require.NoError(t, err) + require.Equal(t, 0, len(a.Config.Inputs())) c = config.NewConfig() c.InputFilters = []string{"mysql", "foo"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 1, len(a.Config.Inputs)) + a = NewAgent(context.Background(), c) + c.SetAgent(a) + err = c.LoadConfig(context.Background(), context.Background(), "../config/testdata/telegraf-agent.toml") + require.NoError(t, err) + require.Equal(t, 1, len(a.Config.Inputs())) c = config.NewConfig() c.InputFilters = []string{"mysql", "redis"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 2, len(a.Config.Inputs)) + a = NewAgent(context.Background(), c) + c.SetAgent(a) + err = c.LoadConfig(context.Background(), context.Background(), "../config/testdata/telegraf-agent.toml") + require.NoError(t, err) + require.Equal(t, 2, len(a.Config.Inputs())) c = config.NewConfig() c.InputFilters = []string{"mysql", "foo", "redis", "bar"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 2, len(a.Config.Inputs)) + a = NewAgent(context.Background(), c) + c.SetAgent(a) + err = c.LoadConfig(context.Background(), context.Background(), "../config/testdata/telegraf-agent.toml") + require.NoError(t, err) + require.Equal(t, 2, len(a.Config.Inputs())) } func TestAgent_LoadOutput(t *testing.T) { - c := config.NewConfig() - c.OutputFilters = []string{"influxdb"} - err := c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ := NewAgent(c) - assert.Equal(t, 2, len(a.Config.Outputs)) - - c = config.NewConfig() - c.OutputFilters = []string{"kafka"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 1, len(a.Config.Outputs)) - - c = config.NewConfig() - c.OutputFilters = []string{} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 3, len(a.Config.Outputs)) - - c = config.NewConfig() - c.OutputFilters = []string{"foo"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 0, len(a.Config.Outputs)) - - c = config.NewConfig() - c.OutputFilters = []string{"influxdb", "foo"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 2, len(a.Config.Outputs)) - - c = config.NewConfig() - c.OutputFilters = []string{"influxdb", "kafka"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - assert.Equal(t, 3, len(c.Outputs)) - a, _ = NewAgent(c) - assert.Equal(t, 3, len(a.Config.Outputs)) - - c = config.NewConfig() - c.OutputFilters = []string{"influxdb", "foo", "kafka", "bar"} - err = c.LoadConfig("../config/testdata/telegraf-agent.toml") - assert.NoError(t, err) - a, _ = NewAgent(c) - assert.Equal(t, 3, len(a.Config.Outputs)) + tests := []struct { + OutputFilters []string + ExpectedOutputs int + }{ + {OutputFilters: []string{"influxdb"}, ExpectedOutputs: 2}, // two instances in toml + {OutputFilters: []string{"kafka"}, ExpectedOutputs: 1}, + {OutputFilters: []string{}, ExpectedOutputs: 3}, + {OutputFilters: []string{"foo"}, ExpectedOutputs: 0}, + {OutputFilters: []string{"influxdb", "foo"}, ExpectedOutputs: 2}, + {OutputFilters: []string{"influxdb", "kafka"}, ExpectedOutputs: 3}, + {OutputFilters: []string{"influxdb", "foo", "kafka", "bar"}, ExpectedOutputs: 3}, + } + for _, test := range tests { + name := strings.Join(test.OutputFilters, "-") + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := config.NewConfig() + c.OutputFilters = test.OutputFilters + a := NewAgent(ctx, c) + c.SetAgent(a) + err := c.LoadConfig(context.Background(), context.Background(), "../config/testdata/telegraf-agent.toml") + require.NoError(t, err) + require.Len(t, a.Config.Outputs(), test.ExpectedOutputs) + }) + } } func TestWindow(t *testing.T) { diff --git a/agent/agentcontrol_test.go b/agent/agentcontrol_test.go new file mode 100644 index 0000000000000..1a2b166057e08 --- /dev/null +++ b/agent/agentcontrol_test.go @@ -0,0 +1,254 @@ +package agent + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/models" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +var now = time.Date(2021, 4, 9, 0, 0, 0, 0, time.UTC) + +func TestAgentPluginControllerLifecycle(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cfg := config.NewConfig() + a := NewAgent(ctx, cfg) + // cfg.SetAgent(a) + inp := &testInputPlugin{} + _ = inp.Init() + ri := models.NewRunningInput(inp, &models.InputConfig{Name: "in"}) + a.AddInput(ri) + go a.RunInput(ri, time.Now()) + + m := metric.New("testing", + map[string]string{ + "country": "canada", + }, + map[string]interface{}{ + "population": 37_590_000, + }, + now) + + rp := models.NewRunningProcessor(&testProcessorPlugin{}, &models.ProcessorConfig{Name: "proc"}) + a.AddProcessor(rp) + go a.RunProcessor(rp) + + outputCtx, outputCancel := context.WithCancel(context.Background()) + o := &testOutputPlugin{} + _ = o.Init() + ro := models.NewRunningOutput(o, &models.OutputConfig{Name: "out"}, 100, 100) + a.AddOutput(ro) + go a.RunOutput(outputCtx, ro) + + go a.RunWithAPI(outputCancel) + + inp.injectMetric(m) + + waitForStatus(t, ri, "running", 1*time.Second) + waitForStatus(t, rp, "running", 1*time.Second) + waitForStatus(t, ro, "running", 1*time.Second) + + cancel() + + waitForStatus(t, ri, "dead", 1*time.Second) + waitForStatus(t, rp, "dead", 1*time.Second) + waitForStatus(t, ro, "dead", 1*time.Second) + + require.Len(t, o.receivedMetrics, 1) + expected := testutil.MustMetric("testing", + map[string]string{ + "country": "canada", + }, + map[string]interface{}{ + "population": 37_590_000, + "capital": "Ottawa", + }, + now) + testutil.RequireMetricEqual(t, expected, o.receivedMetrics[0]) +} + +func TestAgentPluginConnectionsAfterAddAndRemoveProcessor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cfg := config.NewConfig() + a := NewAgent(ctx, cfg) + // cfg.SetAgent(a) + + // start an input + inp := &testInputPlugin{} + _ = inp.Init() + ri := models.NewRunningInput(inp, &models.InputConfig{Name: "in"}) + a.AddInput(ri) + go a.RunInput(ri, time.Now()) + + // start output + outputCtx, outputCancel := context.WithCancel(context.Background()) + o := &testOutputPlugin{} + _ = o.Init() + ro := models.NewRunningOutput(o, &models.OutputConfig{Name: "out"}, 100, 100) + a.AddOutput(ro) + go a.RunOutput(outputCtx, ro) + + // Run agent + go a.RunWithAPI(outputCancel) + + // wait for plugins to start + waitForStatus(t, ri, "running", 1*time.Second) + waitForStatus(t, ro, "running", 1*time.Second) + + // inject a metric into the input plugin as if it collected it + m := metric.New("mojo", nil, map[string]interface{}{"jenkins": "leroy"}, now) + inp.injectMetric(m) + + // wait for the output to get it + o.wait(1) + testutil.RequireMetricEqual(t, m, o.receivedMetrics[0]) + o.clear() + + // spin up new processor + rp := models.NewRunningProcessor(&testProcessorPlugin{}, &models.ProcessorConfig{Name: "proc"}) + a.AddProcessor(rp) + go a.RunProcessor(rp) + + // wait for the processor to start + waitForStatus(t, rp, "running", 5*time.Second) + + // inject a metric into the input + inp.injectMetric(m) + // wait for it to arrive + o.wait(1) + + // create the expected output for comparison + expected := m.Copy() + expected.AddField("capital", "Ottawa") + + testutil.RequireMetricEqual(t, expected, o.receivedMetrics[0]) + + o.clear() + + // stop processor and wait for it to stop + a.StopProcessor(rp) + waitForStatus(t, rp, "dead", 5*time.Second) + + // inject a new metric + inp.injectMetric(m) + + // wait for the output to get it + o.wait(1) + testutil.RequireMetricEqual(t, m, o.receivedMetrics[0]) + o.clear() + + // cancel the app's context + cancel() + + // wait for plugins to stop + waitForStatus(t, ri, "dead", 5*time.Second) + waitForStatus(t, ro, "dead", 5*time.Second) +} + +type hasState interface { + GetState() models.PluginState +} + +func waitForStatus(t *testing.T, stateable hasState, waitStatus string, timeout time.Duration) { + timeoutAt := time.Now().Add(timeout) + for timeoutAt.After(time.Now()) { + if stateable.GetState().String() == waitStatus { + return + } + time.Sleep(10 * time.Millisecond) + } + require.FailNow(t, "timed out waiting for status "+waitStatus) +} + +type testInputPlugin struct { + sync.Mutex + *sync.Cond + started bool + acc telegraf.Accumulator +} + +func (p *testInputPlugin) Init() error { + p.Cond = sync.NewCond(&p.Mutex) + return nil +} +func (p *testInputPlugin) SampleConfig() string { return "" } +func (p *testInputPlugin) Description() string { return "testInputPlugin" } +func (p *testInputPlugin) Gather(a telegraf.Accumulator) error { return nil } +func (p *testInputPlugin) Start(a telegraf.Accumulator) error { + p.Lock() + defer p.Unlock() + p.acc = a + p.started = true + p.Cond.Broadcast() + return nil +} +func (p *testInputPlugin) Stop() { + p.Lock() + defer p.Unlock() +} +func (p *testInputPlugin) injectMetric(m telegraf.Metric) { + p.Lock() + defer p.Unlock() + for !p.started { + p.Cond.Wait() + } + p.acc.AddMetric(m) +} + +type testProcessorPlugin struct { +} + +func (p *testProcessorPlugin) Init() error { return nil } +func (p *testProcessorPlugin) SampleConfig() string { return "" } +func (p *testProcessorPlugin) Description() string { return "testProcessorPlugin" } +func (p *testProcessorPlugin) Start(acc telegraf.Accumulator) error { return nil } +func (p *testProcessorPlugin) Add(m telegraf.Metric, acc telegraf.Accumulator) error { + m.AddField("capital", "Ottawa") + acc.AddMetric(m) + return nil +} +func (p *testProcessorPlugin) Stop() error { return nil } + +type testOutputPlugin struct { + sync.Mutex + *sync.Cond + receivedMetrics []telegraf.Metric +} + +func (p *testOutputPlugin) Init() error { + p.Cond = sync.NewCond(&p.Mutex) + return nil +} +func (p *testOutputPlugin) SampleConfig() string { return "" } +func (p *testOutputPlugin) Description() string { return "testOutputPlugin" } +func (p *testOutputPlugin) Connect() error { return nil } +func (p *testOutputPlugin) Close() error { return nil } +func (p *testOutputPlugin) Write(metrics []telegraf.Metric) error { + p.Lock() + defer p.Unlock() + p.receivedMetrics = append(p.receivedMetrics, metrics...) + p.Broadcast() + return nil +} + +// Wait for the given number of metrics to arrive +func (p *testOutputPlugin) wait(n int) { + p.Lock() + defer p.Unlock() + for len(p.receivedMetrics) < n { + p.Cond.Wait() + } +} + +func (p *testOutputPlugin) clear() { + p.Lock() + defer p.Unlock() + p.receivedMetrics = nil +} diff --git a/agent/tick_test.go b/agent/tick_test.go index 69bf0c2affa39..510af1f10f95f 100644 --- a/agent/tick_test.go +++ b/agent/tick_test.go @@ -34,10 +34,8 @@ func TestAlignedTicker(t *testing.T) { clock.Add(10 * time.Second) for !clock.Now().After(until) { - select { - case tm := <-ticker.Elapsed(): - actual = append(actual, tm.UTC()) - } + tm := <-ticker.Elapsed() + actual = append(actual, tm.UTC()) clock.Add(10 * time.Second) } diff --git a/agenthelper/testAgentController.go b/agenthelper/testAgentController.go new file mode 100644 index 0000000000000..e7087ea3be88c --- /dev/null +++ b/agenthelper/testAgentController.go @@ -0,0 +1,52 @@ +package agenthelper + +// Do not import other Telegraf packages as it causes dependency loops +import ( + "context" + "time" + + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" +) + +// For use in unit tests where the test needs a config object but doesn't +// need to run its plugins. For example when testing parsing toml config. +type TestAgentController struct { + inputs []*models.RunningInput + processors []models.ProcessorRunner + outputs []*models.RunningOutput + // configs []*config.RunningConfigPlugin +} + +func (a *TestAgentController) Reset() { + a.inputs = nil + a.processors = nil + a.outputs = nil + // a.configs = nil +} + +func (a *TestAgentController) RunningInputs() []*models.RunningInput { + return a.inputs +} +func (a *TestAgentController) RunningProcessors() []models.ProcessorRunner { + return a.processors +} +func (a *TestAgentController) RunningOutputs() []*models.RunningOutput { + return a.outputs +} +func (a *TestAgentController) AddInput(input *models.RunningInput) { + a.inputs = append(a.inputs, input) +} +func (a *TestAgentController) AddProcessor(processor models.ProcessorRunner) { + a.processors = append(a.processors, processor) +} +func (a *TestAgentController) AddOutput(output *models.RunningOutput) { + a.outputs = append(a.outputs, output) +} +func (a *TestAgentController) RunInput(input *models.RunningInput, startTime time.Time) {} +func (a *TestAgentController) RunProcessor(p models.ProcessorRunner) {} +func (a *TestAgentController) RunOutput(ctx context.Context, output *models.RunningOutput) {} +func (a *TestAgentController) RunConfigPlugin(ctx context.Context, plugin config.ConfigPlugin) {} +func (a *TestAgentController) StopInput(i *models.RunningInput) {} +func (a *TestAgentController) StopProcessor(p models.ProcessorRunner) {} +func (a *TestAgentController) StopOutput(p *models.RunningOutput) {} diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index 688c1e5bdd6c5..2d6dca4c6b65d 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "flag" "fmt" "log" @@ -23,11 +22,15 @@ import ( "github.com/influxdata/telegraf/internal/goplugin" "github.com/influxdata/telegraf/logger" _ "github.com/influxdata/telegraf/plugins/aggregators/all" + _ "github.com/influxdata/telegraf/plugins/configs" + _ "github.com/influxdata/telegraf/plugins/configs/all" "github.com/influxdata/telegraf/plugins/inputs" _ "github.com/influxdata/telegraf/plugins/inputs/all" "github.com/influxdata/telegraf/plugins/outputs" _ "github.com/influxdata/telegraf/plugins/outputs/all" _ "github.com/influxdata/telegraf/plugins/processors/all" + _ "github.com/influxdata/telegraf/plugins/storage" + _ "github.com/influxdata/telegraf/plugins/storage/all" "gopkg.in/tomb.v1" ) @@ -197,35 +200,34 @@ func runAgent(ctx context.Context, c := config.NewConfig() c.OutputFilters = outputFilters c.InputFilters = inputFilters + + ag := agent.NewAgent(ctx, c) + c.SetAgent(ag) + outputCtx, outputCancel := context.WithCancel(context.Background()) + defer outputCancel() + var err error // providing no "config" flag should load default config if len(fConfigs) == 0 { - err = c.LoadConfig("") + err = c.LoadConfig(ctx, outputCtx, "") if err != nil { return err } } for _, fConfig := range fConfigs { - err = c.LoadConfig(fConfig) + err = c.LoadConfig(ctx, outputCtx, fConfig) if err != nil { return err } } for _, fConfigDirectory := range fConfigDirs { - err = c.LoadDirectory(fConfigDirectory) + err = c.LoadDirectory(ctx, outputCtx, fConfigDirectory) if err != nil { return err } } - if !*fTest && len(c.Outputs) == 0 { - return errors.New("Error: no outputs found, did you provide a valid config file?") - } - if *fPlugins == "" && len(c.Inputs) == 0 { - return errors.New("Error: no inputs found, did you provide a valid config file?") - } - if int64(c.Agent.Interval) <= 0 { return fmt.Errorf("Agent interval must be positive, found %v", c.Agent.Interval) } @@ -234,11 +236,6 @@ func runAgent(ctx context.Context, return fmt.Errorf("Agent flush_interval must be positive; found %v", c.Agent.Interval) } - ag, err := agent.NewAgent(c) - if err != nil { - return err - } - // Setup logging as configured. telegraf.Debug = ag.Config.Agent.Debug || *fDebug logConfig := logger.LogConfig{ @@ -255,13 +252,13 @@ func runAgent(ctx context.Context, logger.SetupLogging(logConfig) if *fRunOnce { - wait := time.Duration(*fTestWait) * time.Second - return ag.Once(ctx, wait) + // wait := time.Duration(*fTestWait) * time.Second + // return ag.Once(ctx, wait) } if *fTest || *fTestWait != 0 { - wait := time.Duration(*fTestWait) * time.Second - return ag.Test(ctx, wait) + // wait := time.Duration(*fTestWait) * time.Second + // return ag.Test(ctx, wait) } log.Printf("I! Loaded inputs: %s", strings.Join(c.InputNames(), " ")) @@ -288,7 +285,8 @@ func runAgent(ctx context.Context, } } - return ag.Run(ctx) + ag.RunWithAPI(outputCancel) + return nil } func usageExit(rc int) { diff --git a/config/config.go b/config/config.go index 56beed8ee4910..f830034efe387 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "context" "fmt" "io/ioutil" "log" @@ -51,7 +52,7 @@ var ( `"`, `\"`, `\`, `\\`, ) - httpLoadConfigRetryInterval = 10 * time.Second + HTTPLoadConfigRetryInterval = 10 * time.Second // fetchURLRe is a regex to determine whether the requested file should // be fetched from a remote or read from the filesystem. @@ -70,13 +71,10 @@ type Config struct { InputFilters []string OutputFilters []string - Agent *AgentConfig - Inputs []*models.RunningInput - Outputs []*models.RunningOutput - Aggregators []*models.RunningAggregator - // Processors have a slice wrapper type because they need to be sorted - Processors models.RunningProcessors - AggProcessors models.RunningProcessors + Agent *AgentConfig + controller AgentController + // ConfigPlugins []ConfigPlugin + // StoragePlugins []StoragePlugin } // NewConfig creates a new struct to hold the Telegraf config. @@ -96,10 +94,6 @@ func NewConfig() *Config { }, Tags: make(map[string]string), - Inputs: make([]*models.RunningInput, 0), - Outputs: make([]*models.RunningOutput, 0), - Processors: make([]*models.RunningProcessor, 0), - AggProcessors: make([]*models.RunningProcessor, 0), InputFilters: make([]string, 0), OutputFilters: make([]string, 0), } @@ -110,7 +104,6 @@ func NewConfig() *Config { MissingField: c.missingTomlField, } c.toml = tomlCfg - return c } @@ -204,7 +197,7 @@ type AgentConfig struct { // InputNames returns a list of strings of the configured inputs. func (c *Config) InputNames() []string { var name []string - for _, input := range c.Inputs { + for _, input := range c.controller.RunningInputs() { name = append(name, input.Config.Name) } return PluginNameCounts(name) @@ -213,8 +206,10 @@ func (c *Config) InputNames() []string { // AggregatorNames returns a list of strings of the configured aggregators. func (c *Config) AggregatorNames() []string { var name []string - for _, aggregator := range c.Aggregators { - name = append(name, aggregator.Config.Name) + for _, processor := range c.controller.RunningProcessors() { + if p, ok := processor.(*models.RunningAggregator); ok { + name = append(name, p.Config.Name) + } } return PluginNameCounts(name) } @@ -222,8 +217,10 @@ func (c *Config) AggregatorNames() []string { // ProcessorNames returns a list of strings of the configured processors. func (c *Config) ProcessorNames() []string { var name []string - for _, processor := range c.Processors { - name = append(name, processor.Config.Name) + for _, processor := range c.controller.RunningProcessors() { + if p, ok := processor.(*models.RunningProcessor); ok { + name = append(name, p.Config.Name) + } } return PluginNameCounts(name) } @@ -231,12 +228,16 @@ func (c *Config) ProcessorNames() []string { // OutputNames returns a list of strings of the configured outputs. func (c *Config) OutputNames() []string { var name []string - for _, output := range c.Outputs { + for _, output := range c.controller.RunningOutputs() { name = append(name, output.Config.Name) } return PluginNameCounts(name) } +func (c *Config) SetAgent(agent AgentController) { + c.controller = agent +} + // PluginNameCounts returns a list of sorted plugin names and their count func PluginNameCounts(plugins []string) []string { names := make(map[string]int) @@ -285,7 +286,6 @@ var header = `# Telegraf Configuration # Environment variables can be used anywhere in this config file, simply surround # them with ${}. For strings the variable must be within quotes (ie, "${STR_VAR}"), # for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR}) - ` var globalTagsConfig = ` # Global tags can be specified here in key="value" format. @@ -294,7 +294,6 @@ var globalTagsConfig = ` # rack = "1a" ## Environment variables can be used as tags, and throughout the config file # user = "$USER" - ` var agentConfig = ` # Configuration for telegraf agent @@ -379,35 +378,30 @@ var outputHeader = ` ############################################################################### # OUTPUT PLUGINS # ############################################################################### - ` var processorHeader = ` ############################################################################### # PROCESSOR PLUGINS # ############################################################################### - ` var aggregatorHeader = ` ############################################################################### # AGGREGATOR PLUGINS # ############################################################################### - ` var inputHeader = ` ############################################################################### # INPUT PLUGINS # ############################################################################### - ` var serviceInputHeader = ` ############################################################################### # SERVICE INPUT PLUGINS # ############################################################################### - ` // PrintSampleConfig prints the sample config @@ -418,8 +412,8 @@ func PrintSampleConfig( aggregatorFilters []string, processorFilters []string, ) { - // print headers - fmt.Printf(header) + // nolint:revive // print headers + fmt.Println(header) if len(sectionFilters) == 0 { sectionFilters = sectionDefaults @@ -430,11 +424,13 @@ func PrintSampleConfig( if sliceContains("outputs", sectionFilters) { if len(outputFilters) != 0 { if len(outputFilters) >= 3 && outputFilters[1] != "none" { - fmt.Printf(outputHeader) + // nolint:revive + fmt.Println(outputHeader) } printFilteredOutputs(outputFilters, false) } else { - fmt.Printf(outputHeader) + // nolint:revive + fmt.Println(outputHeader) printFilteredOutputs(outputDefaults, false) // Print non-default outputs, commented var pnames []string @@ -452,11 +448,13 @@ func PrintSampleConfig( if sliceContains("processors", sectionFilters) { if len(processorFilters) != 0 { if len(processorFilters) >= 3 && processorFilters[1] != "none" { - fmt.Printf(processorHeader) + // nolint:revive + fmt.Println(processorHeader) } printFilteredProcessors(processorFilters, false) } else { - fmt.Printf(processorHeader) + // nolint:revive + fmt.Println(processorHeader) pnames := []string{} for pname := range processors.Processors { pnames = append(pnames, pname) @@ -470,11 +468,13 @@ func PrintSampleConfig( if sliceContains("aggregators", sectionFilters) { if len(aggregatorFilters) != 0 { if len(aggregatorFilters) >= 3 && aggregatorFilters[1] != "none" { - fmt.Printf(aggregatorHeader) + // nolint:revive + fmt.Println(aggregatorHeader) } printFilteredAggregators(aggregatorFilters, false) } else { - fmt.Printf(aggregatorHeader) + // nolint:revive + fmt.Println(aggregatorHeader) pnames := []string{} for pname := range aggregators.Aggregators { pnames = append(pnames, pname) @@ -488,11 +488,13 @@ func PrintSampleConfig( if sliceContains("inputs", sectionFilters) { if len(inputFilters) != 0 { if len(inputFilters) >= 3 && inputFilters[1] != "none" { - fmt.Printf(inputHeader) + // nolint:revive + fmt.Println(inputHeader) } printFilteredInputs(inputFilters, false) } else { - fmt.Printf(inputHeader) + // nolint:revive + fmt.Println(inputHeader) printFilteredInputs(inputDefaults, false) // Print non-default inputs, commented var pnames []string @@ -582,7 +584,8 @@ func printFilteredInputs(inputFilters []string, commented bool) { } sort.Strings(servInputNames) - fmt.Printf(serviceInputHeader) + // nolint:revive + fmt.Println(serviceInputHeader) for _, name := range servInputNames { printConfig(name, servInputs[name], "inputs", commented) } @@ -608,11 +611,13 @@ func printFilteredOutputs(outputFilters []string, commented bool) { func printFilteredGlobalSections(sectionFilters []string) { if sliceContains("global_tags", sectionFilters) { - fmt.Printf(globalTagsConfig) + // nolint:revive + fmt.Println(globalTagsConfig) } if sliceContains("agent", sectionFilters) { - fmt.Printf(agentConfig) + // nolint:revive + fmt.Println(agentConfig) } } @@ -669,7 +674,7 @@ func PrintOutputConfig(name string) error { } // LoadDirectory loads all toml config files found in the specified path, recursively. -func (c *Config) LoadDirectory(path string) error { +func (c *Config) LoadDirectory(ctx context.Context, outputCtx context.Context, path string) error { // nolint:revive walkfn := func(thispath string, info os.FileInfo, _ error) error { if info == nil { log.Printf("W! Telegraf is not permitted to read %s", thispath) @@ -688,7 +693,7 @@ func (c *Config) LoadDirectory(path string) error { if len(name) < 6 || name[len(name)-5:] != ".conf" { return nil } - err := c.LoadConfig(thispath) + err := c.LoadConfig(ctx, outputCtx, thispath) if err != nil { return err } @@ -702,7 +707,7 @@ func (c *Config) LoadDirectory(path string) error { // 2. $HOME/.telegraf/telegraf.conf // 3. /etc/telegraf/telegraf.conf // -func getDefaultConfigPath() (string, error) { +func GetDefaultConfigPath() (string, error) { envfile := os.Getenv("TELEGRAF_CONFIG_PATH") homefile := os.ExpandEnv("${HOME}/.telegraf/telegraf.conf") etcfile := "/etc/telegraf/telegraf.conf" @@ -736,10 +741,10 @@ func isURL(str string) bool { } // LoadConfig loads the given config file and applies it to c -func (c *Config) LoadConfig(path string) error { +func (c *Config) LoadConfig(ctx context.Context, outputCtx context.Context, path string) error { // nolint:revive var err error if path == "" { - if path, err = getDefaultConfigPath(); err != nil { + if path, err = GetDefaultConfigPath(); err != nil { return err } } @@ -748,14 +753,14 @@ func (c *Config) LoadConfig(path string) error { return fmt.Errorf("Error loading config file %s: %w", path, err) } - if err = c.LoadConfigData(data); err != nil { + if err = c.LoadConfigData(ctx, outputCtx, data); err != nil { return fmt.Errorf("Error loading config file %s: %w", path, err) } return nil } // LoadConfigData loads TOML-formatted config data -func (c *Config) LoadConfigData(data []byte) error { +func (c *Config) LoadConfigData(ctx context.Context, outputCtx context.Context, data []byte) error { // nolint:revive tbl, err := parseConfig(data) if err != nil { return fmt.Errorf("Error parsing data: %s", err) @@ -816,12 +821,12 @@ func (c *Config) LoadConfigData(data []byte) error { switch pluginSubTable := pluginVal.(type) { // legacy [outputs.influxdb] support case *ast.Table: - if err = c.addOutput(pluginName, pluginSubTable); err != nil { + if err = c.addOutput(outputCtx, pluginName, pluginSubTable); err != nil { return fmt.Errorf("error parsing %s, %w", pluginName, err) } case []*ast.Table: for _, t := range pluginSubTable { - if err = c.addOutput(pluginName, t); err != nil { + if err = c.addOutput(outputCtx, pluginName, t); err != nil { return fmt.Errorf("error parsing %s array, %w", pluginName, err) } } @@ -838,12 +843,12 @@ func (c *Config) LoadConfigData(data []byte) error { switch pluginSubTable := pluginVal.(type) { // legacy [inputs.cpu] support case *ast.Table: - if err = c.addInput(pluginName, pluginSubTable); err != nil { + if err = c.addInput(ctx, pluginName, pluginSubTable); err != nil { return fmt.Errorf("error parsing %s, %w", pluginName, err) } case []*ast.Table: for _, t := range pluginSubTable { - if err = c.addInput(pluginName, t); err != nil { + if err = c.addInput(ctx, pluginName, t); err != nil { return fmt.Errorf("error parsing %s, %w", pluginName, err) } } @@ -889,19 +894,32 @@ func (c *Config) LoadConfigData(data []byte) error { return fmt.Errorf("plugin %s.%s: line %d: configuration specified the fields %q, but they weren't used", name, pluginName, subTable.Line, keys(c.UnusedFields)) } } + case "config": + for pluginName, pluginVal := range subTable.Fields { + switch pluginSubTable := pluginVal.(type) { + case []*ast.Table: + for _, t := range pluginSubTable { + if err = c.addConfigPlugin(ctx, outputCtx, pluginName, t); err != nil { + return fmt.Errorf("Error parsing %s, %s", pluginName, err) + } + } + default: + return fmt.Errorf("Unsupported config format: %s", + pluginName) + } + if len(c.UnusedFields) > 0 { + return fmt.Errorf("plugin %s.%s: line %d: configuration specified the fields %q, but they weren't used", name, pluginName, subTable.Line, keys(c.UnusedFields)) + } + } // Assume it's an input input for legacy config file support if no other // identifiers are present default: - if err = c.addInput(name, subTable); err != nil { + if err = c.addInput(ctx, name, subTable); err != nil { return fmt.Errorf("Error parsing %s, %s", name, err) } } } - if len(c.Processors) > 1 { - sort.Sort(c.Processors) - } - return nil } @@ -957,8 +975,8 @@ func fetchConfig(u *url.URL) ([]byte, error) { if resp.StatusCode != http.StatusOK { if i < retries { - log.Printf("Error getting HTTP config. Retry %d of %d in %s. Status=%d", i, retries, httpLoadConfigRetryInterval, resp.StatusCode) - time.Sleep(httpLoadConfigRetryInterval) + log.Printf("Error getting HTTP config. Retry %d of %d in %s. Status=%d", i, retries, HTTPLoadConfigRetryInterval, resp.StatusCode) + time.Sleep(HTTPLoadConfigRetryInterval) continue } return nil, fmt.Errorf("Retry %d of %d failed to retrieve remote config: %s", i, retries, resp.Status) @@ -1001,6 +1019,44 @@ func parseConfig(contents []byte) (*ast.Table, error) { return toml.Parse(contents) } +// nolint:revive +func (c *Config) addConfigPlugin(ctx context.Context, outputCtx context.Context, name string, table *ast.Table) error { + creator, ok := ConfigPlugins[name] + if !ok { + return fmt.Errorf("Undefined but requested config plugin: %s", name) + } + configPlugin := creator() + + if stCfg, ok := table.Fields["storage"]; ok { + storageCfgTable := stCfg.(*ast.Table) + for name, cfg := range storageCfgTable.Fields { + if spc, ok := StoragePlugins[name]; ok { + sp := spc() + if err := c.toml.UnmarshalTable(cfg.(*ast.Table), sp); err != nil { + return err + } + reflect.ValueOf(configPlugin).Elem().FieldByName("Storage").Set(reflect.ValueOf(sp)) + } + } + delete(table.Fields, "storage") + } + + if err := c.toml.UnmarshalTable(table, configPlugin); err != nil { + return err + } + + logger := models.NewLogger("config", name, name) + models.SetLoggerOnPlugin(configPlugin, logger) + + if err := configPlugin.Init(ctx, outputCtx, c, c.controller); err != nil { + return err + } + + go c.controller.RunConfigPlugin(ctx, configPlugin) + + return nil +} + func (c *Config) addAggregator(name string, table *ast.Table) error { creator, ok := aggregators.Aggregators[name] if !ok { @@ -1017,7 +1073,16 @@ func (c *Config) addAggregator(name string, table *ast.Table) error { return err } - c.Aggregators = append(c.Aggregators, models.NewRunningAggregator(aggregator, conf)) + ra := models.NewRunningAggregator(aggregator, conf) + + if err := ra.Init(); err != nil { + return err + } + + c.controller.AddProcessor(ra) + + go c.controller.RunProcessor(ra) + return nil } @@ -1036,14 +1101,14 @@ func (c *Config) addProcessor(name string, table *ast.Table) error { if err != nil { return err } - c.Processors = append(c.Processors, rf) - // save a copy for the aggregator - rf, err = c.newRunningProcessor(creator, processorConfig, table) - if err != nil { + if err := rf.Init(); err != nil { return err } - c.AggProcessors = append(c.AggProcessors, rf) + + c.controller.AddProcessor(rf) + + go c.controller.RunProcessor(rf) return nil } @@ -1069,7 +1134,7 @@ func (c *Config) newRunningProcessor( return rf, nil } -func (c *Config) addOutput(name string, table *ast.Table) error { +func (c *Config) addOutput(ctx context.Context, name string, table *ast.Table) error { if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) { return nil } @@ -1099,12 +1164,19 @@ func (c *Config) addOutput(name string, table *ast.Table) error { return err } - ro := models.NewRunningOutput(output, outputConfig, c.Agent.MetricBatchSize, c.Agent.MetricBufferLimit) - c.Outputs = append(c.Outputs, ro) + ro := models.NewRunningOutput(output, outputConfig, + c.Agent.MetricBatchSize, c.Agent.MetricBufferLimit) + + if err := ro.Init(); err != nil { + return err + } + + c.controller.AddOutput(ro) + go c.controller.RunOutput(ctx, ro) return nil } -func (c *Config) addInput(name string, table *ast.Table) error { +func (c *Config) addInput(ctx context.Context, name string, table *ast.Table) error { if len(c.InputFilters) > 0 && !sliceContains(name, c.InputFilters) { return nil } @@ -1124,7 +1196,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { if t, ok := input.(parsers.ParserInput); ok { parser, err := c.buildParser(name, table) if err != nil { - return err + return fmt.Errorf("buildParser: %w", err) } t.SetParser(parser) } @@ -1132,7 +1204,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { if t, ok := input.(parsers.ParserFuncInput); ok { config, err := c.getParserConfig(name, table) if err != nil { - return err + return fmt.Errorf("getParserConfig: %w", err) } t.SetParserFunc(func() (parsers.Parser, error) { return parsers.NewParser(config) @@ -1141,7 +1213,7 @@ func (c *Config) addInput(name string, table *ast.Table) error { pluginConfig, err := c.buildInput(name, table) if err != nil { - return err + return fmt.Errorf("buildInput: %w", err) } if err := c.toml.UnmarshalTable(table, input); err != nil { @@ -1150,7 +1222,14 @@ func (c *Config) addInput(name string, table *ast.Table) error { rp := models.NewRunningInput(input, pluginConfig) rp.SetDefaultTags(c.Tags) - c.Inputs = append(c.Inputs, rp) + + if err := rp.Init(); err != nil { + return fmt.Errorf("init: %w", err) + } + c.controller.AddInput(rp) + + go c.controller.RunInput(rp, time.Now()) + return nil } @@ -1173,6 +1252,9 @@ func (c *Config) buildAggregator(name string, tbl *ast.Table) (*models.Aggregato c.getFieldString(tbl, "name_suffix", &conf.MeasurementSuffix) c.getFieldString(tbl, "name_override", &conf.NameOverride) c.getFieldString(tbl, "alias", &conf.Alias) + if !c.getFieldInt64(tbl, "order", &conf.Order) { + conf.Order = -10000000 + int64(tbl.Position.Begin) + } conf.Tags = make(map[string]string) if node, ok := tbl.Fields["tags"]; ok { @@ -1201,7 +1283,9 @@ func (c *Config) buildAggregator(name string, tbl *ast.Table) (*models.Aggregato func (c *Config) buildProcessor(name string, tbl *ast.Table) (*models.ProcessorConfig, error) { conf := &models.ProcessorConfig{Name: name} - c.getFieldInt64(tbl, "order", &conf.Order) + if !c.getFieldInt64(tbl, "order", &conf.Order) { + conf.Order = -10000000 + int64(tbl.Position.Begin) + } c.getFieldString(tbl, "alias", &conf.Alias) if c.hasErrs() { @@ -1536,8 +1620,6 @@ func (c *Config) buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, Filter: filter, } - // TODO: support FieldPass/FieldDrop on outputs - c.getFieldDuration(tbl, "flush_interval", &oc.FlushInterval) c.getFieldDuration(tbl, "flush_jitter", &oc.FlushJitter) @@ -1652,19 +1734,29 @@ func (c *Config) getFieldInt(tbl *ast.Table, fieldName string, target *int) { } } -func (c *Config) getFieldInt64(tbl *ast.Table, fieldName string, target *int64) { - if node, ok := tbl.Fields[fieldName]; ok { - if kv, ok := node.(*ast.KeyValue); ok { - if iAst, ok := kv.Value.(*ast.Integer); ok { - i, err := iAst.Int() - if err != nil { - c.addError(tbl, fmt.Errorf("unexpected int type %q, expecting int", iAst.Value)) - return - } - *target = i - } - } +// getFieldInt64 returns true if the value was defined in config +func (c *Config) getFieldInt64(tbl *ast.Table, fieldName string, target *int64) bool { + node, ok := tbl.Fields[fieldName] + if !ok { + return false } + kv, ok := node.(*ast.KeyValue) + if !ok { + c.addError(tbl, fmt.Errorf("expected key/value pair for node %q, but got %t", fieldName, node)) + return false + } + iAst, ok := kv.Value.(*ast.Integer) + if !ok { + c.addError(tbl, fmt.Errorf("expected int variable type for %q, but got %t", fieldName, kv.Value)) + return false + } + i, err := iAst.Int() + if err != nil { + c.addError(tbl, fmt.Errorf("unexpected int type %q, expecting int", iAst.Value)) + return false + } + *target = i + return true } func (c *Config) getFieldStringSlice(tbl *ast.Table, fieldName string, target *[]string) { @@ -1745,6 +1837,24 @@ func (c *Config) addError(tbl *ast.Table, err error) { c.errs = append(c.errs, fmt.Errorf("line %d:%d: %w", tbl.Line, tbl.Position, err)) } +// Inputs is a convenience method that lets you get a list of running Inputs from the config +// this is supported for historical reasons +func (c *Config) Inputs() []*models.RunningInput { + return c.controller.RunningInputs() +} + +// Processors is a convenience method that lets you get a list of running Processors from the config +// this is supported for historical reasons +func (c *Config) Processors() []models.ProcessorRunner { + return c.controller.RunningProcessors() +} + +// Outputs is a convenience method that lets you get a list of running Outputs from the config +// this is supported for historical reasons +func (c *Config) Outputs() []*models.RunningOutput { + return c.controller.RunningOutputs() +} + // unwrappable lets you retrieve the original telegraf.Processor from the // StreamingProcessor. This is necessary because the toml Unmarshaller won't // look inside composed types. diff --git a/config/config_test.go b/config/config_test.go index 940b84ada7773..66e88bcca5331 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,7 @@ -package config +package config_test import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -11,19 +12,30 @@ import ( "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/agenthelper" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/models" + _ "github.com/influxdata/telegraf/plugins/aggregators/minmax" "github.com/influxdata/telegraf/plugins/common/tls" "github.com/influxdata/telegraf/plugins/inputs" + _ "github.com/influxdata/telegraf/plugins/inputs/file" "github.com/influxdata/telegraf/plugins/outputs" + _ "github.com/influxdata/telegraf/plugins/outputs/file" "github.com/influxdata/telegraf/plugins/parsers" + _ "github.com/influxdata/telegraf/plugins/processors/rename" "github.com/stretchr/testify/require" ) func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { - c := NewConfig() + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() require.NoError(t, os.Setenv("MY_TEST_SERVER", "192.168.1.1")) require.NoError(t, os.Setenv("TEST_INTERVAL", "10s")) - c.LoadConfig("./testdata/single_plugin_env_vars.toml") + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/single_plugin_env_vars.toml") + require.NoError(t, err) input := inputs.Inputs["memcached"]().(*MockupInputPlugin) input.Servers = []string{"192.168.1.1"} @@ -54,16 +66,22 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) { } inputConfig.Tags = make(map[string]string) + require.Len(t, c.Inputs(), 1) + // Ignore Log and Parser - c.Inputs[0].Input.(*MockupInputPlugin).Log = nil - c.Inputs[0].Input.(*MockupInputPlugin).parser = nil - require.Equal(t, input, c.Inputs[0].Input, "Testdata did not produce a correct mockup struct.") - require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct input metadata.") + c.Inputs()[0].Input.(*MockupInputPlugin).Log = nil + c.Inputs()[0].Input.(*MockupInputPlugin).parser = nil + require.Equal(t, input, c.Inputs()[0].Input, "Testdata did not produce a correct mockup struct.") + require.Equal(t, inputConfig, c.Inputs()[0].Config, "Testdata did not produce correct input metadata.") } func TestConfig_LoadSingleInput(t *testing.T) { - c := NewConfig() - c.LoadConfig("./testdata/single_plugin.toml") + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/single_plugin.toml") + require.NoError(t, err) input := inputs.Inputs["memcached"]().(*MockupInputPlugin) input.Servers = []string{"localhost"} @@ -94,17 +112,22 @@ func TestConfig_LoadSingleInput(t *testing.T) { } inputConfig.Tags = make(map[string]string) + require.Len(t, c.Inputs(), 1) + // Ignore Log and Parser - c.Inputs[0].Input.(*MockupInputPlugin).Log = nil - c.Inputs[0].Input.(*MockupInputPlugin).parser = nil - require.Equal(t, input, c.Inputs[0].Input, "Testdata did not produce a correct memcached struct.") - require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct memcached metadata.") + c.Inputs()[0].Input.(*MockupInputPlugin).Log = nil + c.Inputs()[0].Input.(*MockupInputPlugin).parser = nil + require.Equal(t, input, c.Inputs()[0].Input, "Testdata did not produce a correct memcached struct.") + require.Equal(t, inputConfig, c.Inputs()[0].Config, "Testdata did not produce correct memcached metadata.") } func TestConfig_LoadDirectory(t *testing.T) { - c := NewConfig() - require.NoError(t, c.LoadConfig("./testdata/single_plugin.toml")) - require.NoError(t, c.LoadDirectory("./testdata/subconfig")) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + require.NoError(t, c.LoadConfig(context.Background(), context.Background(), "./testdata/single_plugin.toml")) + require.NoError(t, c.LoadDirectory(context.Background(), context.Background(), "./testdata/subconfig")) // Create the expected data expectedPlugins := make([]*MockupInputPlugin, 4) @@ -189,9 +212,9 @@ func TestConfig_LoadDirectory(t *testing.T) { expectedConfigs[3].Tags = make(map[string]string) // Check the generated plugins - require.Len(t, c.Inputs, len(expectedPlugins)) - require.Len(t, c.Inputs, len(expectedConfigs)) - for i, plugin := range c.Inputs { + require.Len(t, c.Inputs(), len(expectedPlugins)) + require.Len(t, c.Inputs(), len(expectedConfigs)) + for i, plugin := range c.Inputs() { input := plugin.Input.(*MockupInputPlugin) // Check the logger and ignore it for comparison require.NotNil(t, input.Log) @@ -208,59 +231,75 @@ func TestConfig_LoadDirectory(t *testing.T) { } func TestConfig_LoadSpecialTypes(t *testing.T) { - c := NewConfig() - require.NoError(t, c.LoadConfig("./testdata/special_types.toml")) - require.Len(t, c.Inputs, 1) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/special_types.toml") + require.NoError(t, err) + require.Equal(t, 1, len(c.Inputs())) - input, ok := c.Inputs[0].Input.(*MockupInputPlugin) + input, ok := c.Inputs()[0].Input.(*MockupInputPlugin) require.True(t, ok) // Tests telegraf duration parsing. - require.Equal(t, Duration(time.Second), input.WriteTimeout) - // Tests telegraf size parsing. - require.Equal(t, Size(1024*1024), input.MaxBodySize) + require.Equal(t, config.Duration(time.Second), input.WriteTimeout) // Tests telegraf size parsing. + require.Equal(t, config.Size(1024*1024), input.MaxBodySize) // Tests toml multiline basic strings. require.Equal(t, "/path/to/my/cert", strings.TrimRight(input.TLSCert, "\r\n")) } func TestConfig_FieldNotDefined(t *testing.T) { - c := NewConfig() - err := c.LoadConfig("./testdata/invalid_field.toml") + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/invalid_field.toml") require.Error(t, err, "invalid field name") require.Equal(t, "Error loading config file ./testdata/invalid_field.toml: plugin inputs.http_listener_v2: line 1: configuration specified the fields [\"not_a_field\"], but they weren't used", err.Error()) } func TestConfig_WrongFieldType(t *testing.T) { - c := NewConfig() - err := c.LoadConfig("./testdata/wrong_field_type.toml") + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/wrong_field_type.toml") require.Error(t, err, "invalid field type") - require.Equal(t, "Error loading config file ./testdata/wrong_field_type.toml: error parsing http_listener_v2, line 2: (config.MockupInputPlugin.Port) cannot unmarshal TOML string into int", err.Error()) + require.Equal(t, "Error loading config file ./testdata/wrong_field_type.toml: error parsing http_listener_v2, line 2: (config_test.MockupInputPlugin.Port) cannot unmarshal TOML string into int", err.Error()) - c = NewConfig() - err = c.LoadConfig("./testdata/wrong_field_type2.toml") + c = config.NewConfig() + c.SetAgent(agentController) + err = c.LoadConfig(context.Background(), context.Background(), "./testdata/wrong_field_type2.toml") require.Error(t, err, "invalid field type2") - require.Equal(t, "Error loading config file ./testdata/wrong_field_type2.toml: error parsing http_listener_v2, line 2: (config.MockupInputPlugin.Methods) cannot unmarshal TOML string into []string", err.Error()) + require.Equal(t, "Error loading config file ./testdata/wrong_field_type2.toml: error parsing http_listener_v2, line 2: (config_test.MockupInputPlugin.Methods) cannot unmarshal TOML string into []string", err.Error()) } func TestConfig_InlineTables(t *testing.T) { // #4098 - c := NewConfig() - require.NoError(t, c.LoadConfig("./testdata/inline_table.toml")) - require.Len(t, c.Outputs, 2) - - output, ok := c.Outputs[1].Output.(*MockupOuputPlugin) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + require.NoError(t, c.LoadConfig(context.Background(), context.Background(), "./testdata/inline_table.toml")) + require.Len(t, c.Outputs(), 2) + + output, ok := c.Outputs()[1].Output.(*MockupOuputPlugin) require.True(t, ok) require.Equal(t, map[string]string{"Authorization": "Token $TOKEN", "Content-Type": "application/json"}, output.Headers) - require.Equal(t, []string{"org_id"}, c.Outputs[0].Config.Filter.TagInclude) + require.Equal(t, []string{"org_id"}, c.Outputs()[0].Config.Filter.TagInclude) } func TestConfig_SliceComment(t *testing.T) { t.Skipf("Skipping until #3642 is resolved") - c := NewConfig() - require.NoError(t, c.LoadConfig("./testdata/slice_comment.toml")) - require.Len(t, c.Outputs, 1) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + require.NoError(t, c.LoadConfig(context.Background(), context.Background(), "./testdata/slice_comment.toml")) + require.Len(t, c.Outputs(), 1) - output, ok := c.Outputs[0].Output.(*MockupOuputPlugin) + output, ok := c.Outputs()[0].Output.(*MockupOuputPlugin) require.True(t, ok) require.Equal(t, []string{"test"}, output.Scopes) } @@ -268,20 +307,26 @@ func TestConfig_SliceComment(t *testing.T) { func TestConfig_BadOrdering(t *testing.T) { // #3444: when not using inline tables, care has to be taken so subsequent configuration // doesn't become part of the table. This is not a bug, but TOML syntax. - c := NewConfig() - err := c.LoadConfig("./testdata/non_slice_slice.toml") + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/non_slice_slice.toml") require.Error(t, err, "bad ordering") require.Equal(t, "Error loading config file ./testdata/non_slice_slice.toml: error parsing http array, line 4: cannot unmarshal TOML array into string (need slice)", err.Error()) } func TestConfig_AzureMonitorNamespacePrefix(t *testing.T) { // #8256 Cannot use empty string as the namespace prefix - c := NewConfig() - require.NoError(t, c.LoadConfig("./testdata/azure_monitor.toml")) - require.Len(t, c.Outputs, 2) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + require.NoError(t, c.LoadConfig(context.Background(), context.Background(), "./testdata/azure_monitor.toml")) + require.Len(t, c.Outputs(), 2) expectedPrefix := []string{"Telegraf/", ""} - for i, plugin := range c.Outputs { + for i, plugin := range c.Outputs() { output, ok := plugin.Output.(*MockupOuputPlugin) require.True(t, ok) require.Equal(t, expectedPrefix[i], output.NamespacePrefix) @@ -289,7 +334,7 @@ func TestConfig_AzureMonitorNamespacePrefix(t *testing.T) { } func TestConfig_URLRetries3Fails(t *testing.T) { - httpLoadConfigRetryInterval = 0 * time.Second + config.HTTPLoadConfigRetryInterval = 0 * time.Second responseCounter := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) @@ -299,15 +344,18 @@ func TestConfig_URLRetries3Fails(t *testing.T) { expected := fmt.Sprintf("Error loading config file %s: Retry 3 of 3 failed to retrieve remote config: 404 Not Found", ts.URL) - c := NewConfig() - err := c.LoadConfig(ts.URL) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), ts.URL) require.Error(t, err) require.Equal(t, expected, err.Error()) require.Equal(t, 4, responseCounter) } func TestConfig_URLRetries3FailsThenPasses(t *testing.T) { - httpLoadConfigRetryInterval = 0 * time.Second + config.HTTPLoadConfigRetryInterval = 0 * time.Second responseCounter := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if responseCounter <= 2 { @@ -319,8 +367,11 @@ func TestConfig_URLRetries3FailsThenPasses(t *testing.T) { })) defer ts.Close() - c := NewConfig() - require.NoError(t, c.LoadConfig(ts.URL)) + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + require.NoError(t, c.LoadConfig(context.Background(), context.Background(), ts.URL)) require.Equal(t, 4, responseCounter) } @@ -330,19 +381,25 @@ func TestConfig_getDefaultConfigPathFromEnvURL(t *testing.T) { })) defer ts.Close() - c := NewConfig() + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() err := os.Setenv("TELEGRAF_CONFIG_PATH", ts.URL) require.NoError(t, err) - configPath, err := getDefaultConfigPath() + configPath, err := config.GetDefaultConfigPath() require.NoError(t, err) require.Equal(t, ts.URL, configPath) - err = c.LoadConfig("") + err = c.LoadConfig(context.Background(), context.Background(), "") require.NoError(t, err) } func TestConfig_URLLikeFileName(t *testing.T) { - c := NewConfig() - err := c.LoadConfig("http:##www.example.com.conf") + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "http:##www.example.com.conf") require.Error(t, err) if runtime.GOOS == "windows" { @@ -353,15 +410,85 @@ func TestConfig_URLLikeFileName(t *testing.T) { } } +func TestConfig_OrderingProcessorsWithAggregators(t *testing.T) { + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/processor_and_aggregator_order.toml") + require.NoError(t, err) + require.Equal(t, 1, len(c.Inputs())) + require.Equal(t, 4, len(c.Processors())) + require.Equal(t, 1, len(c.Outputs())) + + actual := map[string]int64{} + expected := map[string]int64{ + "aggregators.minmax::one": 1, + "processors.rename::two": 2, + "aggregators.minmax::three": 3, + "processors.rename::four": 4, + } + for _, p := range c.Processors() { + actual[p.LogName()] = p.Order() + } + require.EqualValues(t, expected, actual) +} + +func TestConfig_DefaultOrderingProcessorsWithAggregators(t *testing.T) { + c := config.NewConfig() + agentController := &agenthelper.TestAgentController{} + c.SetAgent(agentController) + defer agentController.Reset() + err := c.LoadConfig(context.Background(), context.Background(), "./testdata/processor_and_aggregator_unordered.toml") + require.NoError(t, err) + require.Equal(t, 1, len(c.Inputs())) + require.Equal(t, 4, len(c.Processors())) + require.Equal(t, 1, len(c.Outputs())) + + actual := map[string]int64{} + // negative orders are defaults based on file position order -10,000,000 + expected := map[string]int64{ + "aggregators.minmax::one": -9999984, + "processors.rename::two": -9999945, + "aggregators.minmax::three": -9999907, + "processors.rename::four": 4, + } + for _, p := range c.Processors() { + actual[p.LogName()] = p.Order() + } + require.EqualValues(t, expected, actual) +} + +func TestConfig_OutputFieldPass(t *testing.T) { + now := time.Date(2021, 7, 5, 14, 45, 0, 0, time.UTC) + output := outputs.Outputs["http"]().(*MockupOuputPlugin) + + filter := models.Filter{ + FieldPass: []string{"good"}, + } + require.NoError(t, filter.Compile()) + outputConfig := &models.OutputConfig{ + Name: "http-test", + Filter: filter, + } + ro := models.NewRunningOutput(output, outputConfig, 1, 10) + ro.AddMetric(metric.New("test", map[string]string{"tag": "foo"}, map[string]interface{}{"good": "bar", "ignored": "woo"}, now)) + require.NoError(t, ro.Write()) + ro.Close() + require.Len(t, output.metrics, 1) + require.True(t, output.metrics[0].HasField("good")) + require.False(t, output.metrics[0].HasField("ignored")) +} + /*** Mockup INPUT plugin for testing to avoid cyclic dependencies ***/ type MockupInputPlugin struct { - Servers []string `toml:"servers"` - Methods []string `toml:"methods"` - Timeout Duration `toml:"timeout"` - ReadTimeout Duration `toml:"read_timeout"` - WriteTimeout Duration `toml:"write_timeout"` - MaxBodySize Size `toml:"max_body_size"` - Port int `toml:"port"` + Servers []string `toml:"servers"` + Methods []string `toml:"methods"` + Timeout config.Duration `toml:"timeout"` + ReadTimeout config.Duration `toml:"read_timeout"` + WriteTimeout config.Duration `toml:"write_timeout"` + MaxBodySize config.Size `toml:"max_body_size"` + Port int `toml:"port"` Command string PidFile string Log telegraf.Logger `toml:"-"` @@ -383,18 +510,22 @@ type MockupOuputPlugin struct { NamespacePrefix string `toml:"namespace_prefix"` Log telegraf.Logger `toml:"-"` tls.ClientConfig + metrics []telegraf.Metric } -func (m *MockupOuputPlugin) Connect() error { return nil } -func (m *MockupOuputPlugin) Close() error { return nil } -func (m *MockupOuputPlugin) Description() string { return "Mockup test output plugin" } -func (m *MockupOuputPlugin) SampleConfig() string { return "Mockup test output plugin" } -func (m *MockupOuputPlugin) Write(metrics []telegraf.Metric) error { return nil } +func (m *MockupOuputPlugin) Connect() error { return nil } +func (m *MockupOuputPlugin) Close() error { return nil } +func (m *MockupOuputPlugin) Description() string { return "Mockup test output plugin" } +func (m *MockupOuputPlugin) SampleConfig() string { return "Mockup test output plugin" } +func (m *MockupOuputPlugin) Write(metrics []telegraf.Metric) error { + m.metrics = append(m.metrics, metrics...) + return nil +} // Register the mockup plugin on loading func init() { // Register the mockup input plugin for the required names - inputs.Add("exec", func() telegraf.Input { return &MockupInputPlugin{Timeout: Duration(time.Second * 5)} }) + inputs.Add("exec", func() telegraf.Input { return &MockupInputPlugin{Timeout: config.Duration(time.Second * 5)} }) inputs.Add("http_listener_v2", func() telegraf.Input { return &MockupInputPlugin{} }) inputs.Add("memcached", func() telegraf.Input { return &MockupInputPlugin{} }) inputs.Add("procstat", func() telegraf.Input { return &MockupInputPlugin{} }) diff --git a/config/plugins.go b/config/plugins.go new file mode 100644 index 0000000000000..5dfa774209862 --- /dev/null +++ b/config/plugins.go @@ -0,0 +1,53 @@ +package config + +import ( + "context" + "time" + + "github.com/influxdata/telegraf/models" +) + +type ConfigCreator func() ConfigPlugin +type StorageCreator func() StoragePlugin + +var ( + ConfigPlugins = map[string]ConfigCreator{} + StoragePlugins = map[string]StorageCreator{} +) + +// ConfigPlugin is the interface for implemnting plugins that change how Telegraf works +type ConfigPlugin interface { + GetName() string + Init(ctx context.Context, outputCtx context.Context, cfg *Config, agent AgentController) error + Close() error +} + +// StoragePlugin is the interface to implement if you're building a plugin that implements state storage +type StoragePlugin interface { + Init() error + + Load(namespace, key string, obj interface{}) error + Save(namespace, key string, obj interface{}) error + + Close() error +} + +// AgentController represents the plugin management that the agent is currently doing +type AgentController interface { + RunningInputs() []*models.RunningInput + RunningProcessors() []models.ProcessorRunner + RunningOutputs() []*models.RunningOutput + + AddInput(input *models.RunningInput) + AddProcessor(processor models.ProcessorRunner) + AddOutput(output *models.RunningOutput) + + RunInput(input *models.RunningInput, startTime time.Time) + RunProcessor(p models.ProcessorRunner) + RunOutput(ctx context.Context, output *models.RunningOutput) + RunConfigPlugin(ctx context.Context, plugin ConfigPlugin) + + StopInput(i *models.RunningInput) + StopProcessor(p models.ProcessorRunner) + StopOutput(p *models.RunningOutput) +} diff --git a/config/testdata/processor_and_aggregator_order.toml b/config/testdata/processor_and_aggregator_order.toml new file mode 100644 index 0000000000000..b9496d9e38024 --- /dev/null +++ b/config/testdata/processor_and_aggregator_order.toml @@ -0,0 +1,14 @@ +[[inputs.file]] +[[aggregators.minmax]] + alias = "one" + order = 1 +[[processors.rename]] + alias = "two" + order = 2 +[[aggregators.minmax]] + alias = "three" + order = 3 +[[processors.rename]] + alias = "four" + order = 4 +[[outputs.file]] diff --git a/config/testdata/processor_and_aggregator_unordered.toml b/config/testdata/processor_and_aggregator_unordered.toml new file mode 100644 index 0000000000000..4e98d2fcc9ee9 --- /dev/null +++ b/config/testdata/processor_and_aggregator_unordered.toml @@ -0,0 +1,11 @@ +[[inputs.file]] +[[aggregators.minmax]] + alias = "one" +[[processors.rename]] + alias = "two" +[[aggregators.minmax]] + alias = "three" +[[processors.rename]] + alias = "four" + order = 4 +[[outputs.file]] diff --git a/config/testdata/telegraf-agent.toml b/config/testdata/telegraf-agent.toml index 6967d6e862277..4905415cf7770 100644 --- a/config/testdata/telegraf-agent.toml +++ b/config/testdata/telegraf-agent.toml @@ -154,20 +154,6 @@ ## Offset (must be either "oldest" or "newest") offset = "oldest" -# read metrics from a Kafka legacy topic -[[inputs.kafka_consumer_legacy]] - ## topic(s) to consume - topics = ["telegraf"] - # an array of Zookeeper connection strings - zookeeper_peers = ["localhost:2181"] - ## the name of the consumer group - consumer_group = "telegraf_metrics_consumers" - # Maximum number of points to buffer between collection intervals - point_buffer = 100000 - ## Offset (must be either "oldest" or "newest") - offset = "oldest" - - # Read metrics from a LeoFS Server via SNMP [[inputs.leofs]] # An array of URI to gather stats about LeoFS. @@ -197,14 +183,14 @@ # Metrics groups to be collected, by default, all enabled. master_collections = ["resources","master","system","slaves","frameworks","messages","evqueue","registrar"] -# Read metrics from one or many MongoDB servers -[[inputs.mongodb]] - # An array of URI to gather stats about. Specify an ip or hostname - # with optional port add password. ie mongodb://user:auth_key@10.10.3.30:27017, - # mongodb://10.10.3.33:18832, 10.0.0.1:10000, etc. - # - # If no servers are specified, then 127.0.0.1 is used as the host and 27107 as the port. - servers = ["127.0.0.1:27017"] +# # Read metrics from one or many MongoDB servers +# [[inputs.mongodb]] +# # An array of URI to gather stats about. Specify an ip or hostname +# # with optional port add password. ie mongodb://user:auth_key@10.10.3.30:27017, +# # mongodb://10.10.3.33:18832, 10.0.0.1:10000, etc. +# # +# # If no servers are specified, then 127.0.0.1 is used as the host and 27107 as the port. +# servers = ["127.0.0.1:27017"] # Read metrics from one or many mysql servers [[inputs.mysql]] diff --git a/config/types.go b/config/types.go index 7c1c50b9e3690..eb76fd8641a71 100644 --- a/config/types.go +++ b/config/types.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "fmt" "strconv" "time" @@ -86,3 +87,34 @@ func (s *Size) UnmarshalTOML(b []byte) error { func (s *Size) UnmarshalText(text []byte) error { return s.UnmarshalTOML(text) } + +func (s *Size) MarshalText() []byte { + if s == nil { + return []byte{} + } + v := *s + if v == 0 { + return []byte("0") + } + if v%1024 > 0 { + return []byte(fmt.Sprintf("%dB", v)) + } + v = v / 1024 + if v%1024 > 0 { + return []byte(fmt.Sprintf("%dKiB", v)) + } + v = v / 1024 + if v%1024 > 0 { + return []byte(fmt.Sprintf("%dMiB", v)) + } + v = v / 1024 + if v%1024 > 0 { + return []byte(fmt.Sprintf("%dGiB", v)) + } + v = v / 1024 + if v%1024 > 0 { + return []byte(fmt.Sprintf("%dTiB", v)) + } + v = v / 1024 + return []byte(fmt.Sprintf("%dPiB", v)) +} diff --git a/config/types_test.go b/config/types_test.go index afff599e3d6e4..1eaee932ba475 100644 --- a/config/types_test.go +++ b/config/types_test.go @@ -1,17 +1,21 @@ package config_test import ( + "context" "testing" "time" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" "github.com/influxdata/telegraf/plugins/processors/reverse_dns" "github.com/stretchr/testify/require" ) func TestConfigDuration(t *testing.T) { c := config.NewConfig() - err := c.LoadConfigData([]byte(` + c.SetAgent(&agenthelper.TestAgentController{}) + err := c.LoadConfigData(context.Background(), context.Background(), []byte(` [[processors.reverse_dns]] cache_ttl = "3h" lookup_timeout = "17s" @@ -22,8 +26,8 @@ func TestConfigDuration(t *testing.T) { dest = "source_name" `)) require.NoError(t, err) - require.Len(t, c.Processors, 1) - p := c.Processors[0].Processor.(*reverse_dns.ReverseDNS) + require.Len(t, c.Processors(), 1) + p := c.Processors()[0].(*models.RunningProcessor).Processor.(*reverse_dns.ReverseDNS) require.EqualValues(t, p.CacheTTL, 3*time.Hour) require.EqualValues(t, p.LookupTimeout, 17*time.Second) require.Equal(t, p.MaxParallelLookups, 13) diff --git a/docs/AGGREGATORS_AND_PROCESSORS.md b/docs/AGGREGATORS_AND_PROCESSORS.md index 934a4b0cf7706..b9302baec5c25 100644 --- a/docs/AGGREGATORS_AND_PROCESSORS.md +++ b/docs/AGGREGATORS_AND_PROCESSORS.md @@ -65,3 +65,20 @@ the plugin receives, you can make use of `taginclude` to group aggregates by specific tags only. **Note:** Aggregator plugins only aggregate metrics within their periods (`now() - period`). Data with a timestamp earlier than `now() - period` cannot be included. + + + +------------+ Processors and aggregators can be ordered +--------+ + | Input +---+ and chained arbitrarily +--->+ Output | + +------------+ | | +--------+ + | | + +------------+ | +-----------+ +------------+ +-----------+ +------------+ | +--------+ + | Input +------>+ Processor +--->+ Aggregator +----->+ Processor +--->+ Aggregator +-->---->+ Output | + +------------+ | +-----------+ +------------+ +-----------+ +------------+ | +--------+ + | | + +------------+ | | +--------+ + | Input +---+ +--->+ Output | + +------------+ | | +--------+ + | | +  +------------+ | | +--------+ +  | Input      +---+ +--->+ Output | +  +------------+ +--------+ diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index 7ae13c1143db4..fba8828f7c3f5 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -210,6 +210,7 @@ following works: - github.com/tinylib/msgp [MIT License](https://github.com/tinylib/msgp/blob/master/LICENSE) - github.com/tklauser/go-sysconf [BSD 3-Clause "New" or "Revised" License](https://github.com/tklauser/go-sysconf/blob/master/LICENSE) - github.com/tklauser/numcpus [Apache License 2.0](https://github.com/tklauser/numcpus/blob/master/LICENSE) +- github.com/ugorji/go/codec [MIT License](https://github.com/sleepinggenius2/gosmi/blob/master/LICENSE) - github.com/vapourismo/knx-go [MIT License](https://github.com/vapourismo/knx-go/blob/master/LICENSE) - github.com/vishvananda/netlink [Apache License 2.0](https://github.com/vishvananda/netlink/blob/master/LICENSE) - github.com/vishvananda/netns [Apache License 2.0](https://github.com/vishvananda/netns/blob/master/LICENSE) @@ -225,6 +226,7 @@ following works: - github.com/xdg/stringprep [Apache License 2.0](https://github.com/xdg-go/stringprep/blob/master/LICENSE) - github.com/youmark/pkcs8 [MIT License](https://github.com/youmark/pkcs8/blob/master/LICENSE) - github.com/yuin/gopher-lua [MIT License](https://github.com/yuin/gopher-lua/blob/master/LICENSE) +- go.etcd.io/bbolt [MIT License](https://github.com/sleepinggenius2/gosmi/blob/master/LICENSE) - go.mongodb.org/mongo-driver [Apache License 2.0](https://github.com/mongodb/mongo-go-driver/blob/master/LICENSE) - go.opencensus.io [Apache License 2.0](https://github.com/census-instrumentation/opencensus-go/blob/master/LICENSE) - go.starlark.net [BSD 3-Clause "New" or "Revised" License](https://github.com/google/starlark-go/blob/master/LICENSE) diff --git a/go.mod b/go.mod index f0f36e2df8717..6a147c6773a79 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/google/go-cmp v0.5.5 github.com/google/go-github/v32 v32.1.0 github.com/gopcua/opcua v0.1.13 - github.com/gorilla/mux v1.7.3 + github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/gosnmp/gosnmp v1.32.0 github.com/grid-x/modbus v0.0.0-20210224155242-c4a3d042e99b @@ -128,6 +128,7 @@ require ( github.com/tidwall/gjson v1.8.0 github.com/tinylib/msgp v1.1.5 github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/ugorji/go/codec v1.2.6 github.com/vapourismo/knx-go v0.0.0-20201122213738-75fe09ace330 github.com/vjeantet/grok v1.0.1 github.com/vmware/govmomi v0.19.0 @@ -137,6 +138,7 @@ require ( github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 // indirect + go.etcd.io/bbolt v1.3.6 go.mongodb.org/mongo-driver v1.5.3 go.starlark.net v0.0.0-20210406145628-7a1108eaa012 go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 20b7759feef97..0dfccc49e3a73 100644 --- a/go.sum +++ b/go.sum @@ -782,8 +782,9 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -1484,6 +1485,10 @@ github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+l github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E= +github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0= +github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= +github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1544,6 +1549,8 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1811,6 +1818,7 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/int_test.sh b/int_test.sh new file mode 100755 index 0000000000000..325ad59f0d551 --- /dev/null +++ b/int_test.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +#exit script if any command/pipeline fails +set -e + +#unset variables are errors +set -u + +#return value of a pipeline is the last non-zero return +set -o pipefail + +#print commands before they're executed +#set -x + + +file=telegraf.conf.int_test +port=7551 + +function log() +{ + echo ------- $* +} + +log writing temp config file +tee $file < /dev/null; do + log waiting for listening port + sleep 2 +done + +#it doesn't accept connections for a while after it starts listening +#sleep 10 + +log rest create outputs.file +curl \ + -d '{"name": "outputs.file", "config": {"files": ["stdout"]} }' \ + -H "Content-Type: application/json" \ + -X POST \ + localhost:$port/plugins/create +echo + +log rest create inputs.cpu +curl \ + -d '{"name": "inputs.cpu" }' \ + -H "Content-Type: application/json" \ + -X POST \ + localhost:$port/plugins/create +echo + +function running() +{ + ps $pid > /dev/null + local ret=$? + #echo $ret + return $ret +} + +log make sure it''s still running +running + +log send SIGINT +kill $pid + +log wait for it to exit +sleep 2 + +log make sure it has exited +[ ! running ] + +#unhook the cleanup trap +trap EXIT + +#remove the temp config file +rm $file + +log success diff --git a/internal/channel/relay.go b/internal/channel/relay.go new file mode 100644 index 0000000000000..624e673da8099 --- /dev/null +++ b/internal/channel/relay.go @@ -0,0 +1,50 @@ +package channel + +import ( + "sync" + + "github.com/influxdata/telegraf" +) + +// Relay metrics from one channel to another. Connects a source channel to a destination channel +// Think of it as connecting two pipes together. +// dst is closed when src is closed and the last message has been written out to dst. +type Relay struct { + sync.Mutex + src <-chan telegraf.Metric + dst chan<- telegraf.Metric +} + +func NewRelay(src <-chan telegraf.Metric, dst chan<- telegraf.Metric) *Relay { + return &Relay{ + src: src, + dst: dst, + } +} + +func (r *Relay) Start() { + go func() { + for m := range r.src { + r.GetDest() <- m + } + close(r.GetDest()) + }() +} + +// SetDest changes the destination channel. this is to make channels hot-swappable, kind of like adding or removing items in a linked list. +// Should not be called after the source channel closes. +func (r *Relay) SetDest(dst chan<- telegraf.Metric) { + r.Lock() + defer r.Unlock() + r.dst = dst +} + +// GetDest is the current dst channel +func (r *Relay) GetDest() chan<- telegraf.Metric { + r.Lock() + defer r.Unlock() + if r.dst == nil { + panic("dst channel should never be nil") + } + return r.dst +} diff --git a/internal/channel/relay_test.go b/internal/channel/relay_test.go new file mode 100644 index 0000000000000..891b14871cc60 --- /dev/null +++ b/internal/channel/relay_test.go @@ -0,0 +1,63 @@ +package channel + +import ( + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestUnbufferedConnection(t *testing.T) { + src := make(chan telegraf.Metric) + dst := make(chan telegraf.Metric) + ch := NewRelay(src, dst) + ch.Start() + m := testutil.MustMetric("test", nil, nil, time.Now()) + go func() { + src <- m + close(src) + }() + m2 := <-dst + testutil.RequireMetricEqual(t, m, m2) +} + +func TestBufferedConnection(t *testing.T) { + src := make(chan telegraf.Metric, 1) + dst := make(chan telegraf.Metric, 1) + ch := NewRelay(src, dst) + ch.Start() + m := testutil.MustMetric("test", nil, nil, time.Now()) + src <- m + close(src) + m2 := <-dst + testutil.RequireMetricEqual(t, m, m2) +} + +func TestCloseIsRelayed(t *testing.T) { + src := make(chan telegraf.Metric, 1) + dst := make(chan telegraf.Metric, 1) + ch := NewRelay(src, dst) + ch.Start() + close(src) + _, ok := <-dst + require.False(t, ok) +} + +func TestCanReassignDest(t *testing.T) { + src := make(chan telegraf.Metric, 1) + dst1 := make(chan telegraf.Metric, 1) + dst2 := make(chan telegraf.Metric, 1) + ch := NewRelay(src, dst1) + ch.Start() + m1 := testutil.MustMetric("test1", nil, nil, time.Now()) + m2 := testutil.MustMetric("test2", nil, nil, time.Now()) + src <- m1 + m3 := <-dst1 + ch.SetDest(dst2) + src <- m2 + m4 := <-dst2 + testutil.RequireMetricEqual(t, m1, m3) + testutil.RequireMetricEqual(t, m2, m4) +} diff --git a/internal/rotate/file_writer.go b/internal/rotate/file_writer.go index 7cfde02692cd4..6f3e29993ec59 100644 --- a/internal/rotate/file_writer.go +++ b/internal/rotate/file_writer.go @@ -4,6 +4,7 @@ package rotate import ( "fmt" "io" + "log" // nolint:revive "os" "path/filepath" "sort" @@ -131,7 +132,7 @@ func (w *FileWriter) rotateIfNeeded() error { (w.maxSizeInBytes > 0 && w.bytesWritten >= w.maxSizeInBytes) { if err := w.rotate(); err != nil { //Ignore rotation errors and keep the log open - fmt.Printf("unable to rotate the file '%s', %s", w.filename, err.Error()) + log.Printf("E! [agent] unable to rotate the file '%s', %s", w.filename, err.Error()) } return w.openCurrent() } diff --git a/metric.go b/metric.go index 23098bb8bc71e..f9f12419f6f29 100644 --- a/metric.go +++ b/metric.go @@ -119,5 +119,5 @@ type Metric interface { // Drop marks the metric as processed successfully without being written // to any output. - Drop() + Drop() // TODO: Rename to Ack() ? } diff --git a/models/filter.go b/models/filter.go index 8103c23173297..988a895e7d0b6 100644 --- a/models/filter.go +++ b/models/filter.go @@ -16,22 +16,22 @@ type TagFilter struct { // Filter containing drop/pass and tagdrop/tagpass rules type Filter struct { - NameDrop []string + NameDrop []string `toml:"namedrop"` nameDrop filter.Filter - NamePass []string + NamePass []string `toml:"namepass"` namePass filter.Filter - FieldDrop []string + FieldDrop []string `toml:"fielddrop"` fieldDrop filter.Filter - FieldPass []string + FieldPass []string `toml:"fieldpass"` fieldPass filter.Filter - TagDrop []TagFilter - TagPass []TagFilter + TagDrop []TagFilter `toml:"tagdrop"` + TagPass []TagFilter `toml:"tagpass"` - TagExclude []string + TagExclude []string `toml:"tagexclude"` tagExclude filter.Filter - TagInclude []string + TagInclude []string `toml:"taginclude"` tagInclude filter.Filter isActive bool diff --git a/models/id.go b/models/id.go new file mode 100644 index 0000000000000..624b7911ca3d8 --- /dev/null +++ b/models/id.go @@ -0,0 +1,17 @@ +package models + +import ( + "math/rand" + "sync/atomic" +) + +var ( + globalPluginIDIncrement uint32 + globalInstanceID = rand.Uint32() // set a new instance ID on every app load. +) + +// NextPluginID generates a globally unique plugin ID for use referencing the plugin within the lifetime of Telegraf. +func NextPluginID() uint64 { + num := atomic.AddUint32(&globalPluginIDIncrement, 1) + return uint64(globalInstanceID)<<32 + uint64(num) +} diff --git a/models/running_aggregator.go b/models/running_aggregator.go index 5aa3979c36926..2a48bf99b4ad9 100644 --- a/models/running_aggregator.go +++ b/models/running_aggregator.go @@ -1,16 +1,20 @@ package models import ( + "context" "sync" "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/selfstat" ) type RunningAggregator struct { sync.Mutex + + ID uint64 Aggregator telegraf.Aggregator Config *AggregatorConfig periodStart time.Time @@ -21,6 +25,13 @@ type RunningAggregator struct { MetricsFiltered selfstat.Stat MetricsDropped selfstat.Stat PushTime selfstat.Stat + State + + RoundInterval bool // comes from config + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup } func NewRunningAggregator(aggregator telegraf.Aggregator, config *AggregatorConfig) *RunningAggregator { @@ -37,7 +48,8 @@ func NewRunningAggregator(aggregator telegraf.Aggregator, config *AggregatorConf SetLoggerOnPlugin(aggregator, logger) - return &RunningAggregator{ + a := &RunningAggregator{ + ID: NextPluginID(), Aggregator: aggregator, Config: config, MetricsPushed: selfstat.Register( @@ -62,22 +74,29 @@ func NewRunningAggregator(aggregator telegraf.Aggregator, config *AggregatorConf ), log: logger, } + a.setState(PluginStateCreated) + return a } // AggregatorConfig is the common config for all aggregators. type AggregatorConfig struct { - Name string - Alias string - DropOriginal bool - Period time.Duration - Delay time.Duration - Grace time.Duration + Name string `toml:"name"` + Alias string `toml:"alias"` + DropOriginal bool `toml:"drop_original"` + Period time.Duration `toml:"period"` + Delay time.Duration `toml:"delay"` + Grace time.Duration `toml:"grace"` - NameOverride string - MeasurementPrefix string - MeasurementSuffix string - Tags map[string]string - Filter Filter + NameOverride string `toml:"name_override"` + MeasurementPrefix string `toml:"measurement_prefix"` + MeasurementSuffix string `toml:"measurement_suffix"` + Tags map[string]string `toml:"tags"` + Filter Filter `toml:"filter"` + Order int64 `toml:"order"` +} + +func (r *RunningAggregator) Order() int64 { + return r.Config.Order } func (r *RunningAggregator) LogName() string { @@ -94,14 +113,6 @@ func (r *RunningAggregator) Init() error { return nil } -func (r *RunningAggregator) Period() time.Duration { - return r.Config.Period -} - -func (r *RunningAggregator) EndPeriod() time.Time { - return r.periodEnd -} - func (r *RunningAggregator) UpdateWindow(start, until time.Time) { r.periodStart = start r.periodEnd = until @@ -109,24 +120,20 @@ func (r *RunningAggregator) UpdateWindow(start, until time.Time) { } func (r *RunningAggregator) MakeMetric(metric telegraf.Metric) telegraf.Metric { - m := makemetric( - metric, - r.Config.NameOverride, - r.Config.MeasurementPrefix, - r.Config.MeasurementSuffix, - r.Config.Tags, - nil) - - r.MetricsPushed.Incr(1) - - return m + return metric } // Add a metric to the aggregator and return true if the original metric // should be dropped. -func (r *RunningAggregator) Add(m telegraf.Metric) bool { +func (r *RunningAggregator) Add(m telegraf.Metric, acc telegraf.Accumulator) error { + defer func() { + if !r.Config.DropOriginal && len(m.FieldList()) > 0 { + acc.AddMetric(m) + } + }() + if ok := r.Config.Filter.Select(m); !ok { - return false + return nil } // Make a copy of the metric but don't retain tracking. We do not fail a @@ -138,21 +145,23 @@ func (r *RunningAggregator) Add(m telegraf.Metric) bool { r.Config.Filter.Modify(m) if len(m.FieldList()) == 0 { r.MetricsFiltered.Incr(1) - return r.Config.DropOriginal + m.Drop() + return nil } r.Lock() defer r.Unlock() + // check if outside agg window if m.Time().Before(r.periodStart.Add(-r.Config.Grace)) || m.Time().After(r.periodEnd.Add(r.Config.Delay)) { r.log.Debugf("Metric is outside aggregation window; discarding. %s: m: %s e: %s g: %s", m.Time(), r.periodStart, r.periodEnd, r.Config.Grace) r.MetricsDropped.Incr(1) - return r.Config.DropOriginal + return nil } r.Aggregator.Add(m) - return r.Config.DropOriginal + return nil } func (r *RunningAggregator) Push(acc telegraf.Accumulator) { @@ -163,10 +172,32 @@ func (r *RunningAggregator) Push(acc telegraf.Accumulator) { until := r.periodEnd.Add(r.Config.Period) r.UpdateWindow(since, until) - r.push(acc) + r.push(acc.WithNewMetricMaker(r.LogName(), r.Log(), r.pushMetricMaker)) r.Aggregator.Reset() } +// not passed on to agg at this time +func (r *RunningAggregator) Start(acc telegraf.Accumulator) error { + r.setState(PluginStateRunning) + + since, until := r.calculateUpdateWindow(time.Now()) + r.UpdateWindow(since, until) + + r.ctx, r.cancel = context.WithCancel(context.Background()) + r.wg.Add(1) + go r.pushLoop(acc) + + return nil +} + +// not passed on to agg at this time +func (r *RunningAggregator) Stop() { + r.setState(PluginStateStopping) + r.cancel() + r.wg.Wait() + r.setState(PluginStateDead) +} + func (r *RunningAggregator) push(acc telegraf.Accumulator) { start := time.Now() r.Aggregator.Push(acc) @@ -177,3 +208,58 @@ func (r *RunningAggregator) push(acc telegraf.Accumulator) { func (r *RunningAggregator) Log() telegraf.Logger { return r.log } + +// Before calling Add, initialize the aggregation window. This ensures +// that any metric created after start time will be aggregated. +func (r *RunningAggregator) calculateUpdateWindow(start time.Time) (since time.Time, until time.Time) { + if r.RoundInterval { + until = internal.AlignTime(start, r.Config.Period) + if until == start { + until = internal.AlignTime(start.Add(time.Nanosecond), r.Config.Period) + } + } else { + until = start.Add(r.Config.Period) + } + + since = until.Add(-r.Config.Period) + + return since, until +} + +func (r *RunningAggregator) pushLoop(acc telegraf.Accumulator) { + for { + // Ensures that Push will be called for each period, even if it has + // already elapsed before this function is called. This is guaranteed + // because so long as only Push updates the EndPeriod. This method + // also avoids drift by not using a ticker. + until := time.Until(r.periodEnd) + + select { + case <-time.After(until): + r.Push(acc) + break + case <-r.ctx.Done(): + r.Push(acc) + r.wg.Done() + return + } + } +} + +func (r *RunningAggregator) pushMetricMaker(inMetric telegraf.Metric) telegraf.Metric { + m := makemetric( + inMetric, + r.Config.NameOverride, + r.Config.MeasurementPrefix, + r.Config.MeasurementSuffix, + r.Config.Tags, + nil) + + r.MetricsPushed.Incr(1) + + return m +} + +func (r *RunningAggregator) GetID() uint64 { + return r.ID +} diff --git a/models/running_aggregator_test.go b/models/running_aggregator_test.go index a858859652acf..2d180b795adcc 100644 --- a/models/running_aggregator_test.go +++ b/models/running_aggregator_test.go @@ -16,10 +16,11 @@ func TestAdd(t *testing.T) { Filter: Filter{ NamePass: []string{"*"}, }, - Period: time.Millisecond * 500, + DropOriginal: true, + Period: time.Millisecond * 500, }) require.NoError(t, ra.Config.Filter.Compile()) - acc := testutil.Accumulator{} + acc := &testutil.Accumulator{} now := time.Now() ra.UpdateWindow(now, now.Add(ra.Config.Period)) @@ -31,13 +32,49 @@ func TestAdd(t *testing.T) { }, time.Now().Add(time.Millisecond*150), telegraf.Untyped) - require.False(t, ra.Add(m)) - ra.Push(&acc) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) + ra.Push(acc) require.Equal(t, 1, len(acc.Metrics)) require.Equal(t, int64(101), acc.Metrics[0].Fields["sum"]) } +func TestAddWithoutDrop(t *testing.T) { + a := &TestAggregator{} + ra := NewRunningAggregator(a, &AggregatorConfig{ + Name: "TestRunningAggregator", + Filter: Filter{ + NamePass: []string{"*"}, + }, + DropOriginal: false, + Period: time.Millisecond * 500, + }) + require.NoError(t, ra.Config.Filter.Compile()) + acc := &testutil.Accumulator{} + + now := time.Now() + ra.UpdateWindow(now, now.Add(ra.Config.Period)) + + m := testutil.MustMetric("RITest", + map[string]string{}, + map[string]interface{}{ + "value": int64(101), + }, + time.Now().Add(time.Millisecond*150), + telegraf.Untyped) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) + ra.Push(acc) + + require.Equal(t, 2, len(acc.Metrics)) + metric := acc.Metrics[0] + if _, ok := metric.Fields["sum"]; !ok { + metric = acc.Metrics[1] + } + require.Equal(t, int64(101), metric.Fields["sum"]) +} + func TestAddMetricsOutsideCurrentPeriod(t *testing.T) { a := &TestAggregator{} ra := NewRunningAggregator(a, &AggregatorConfig{ @@ -45,10 +82,11 @@ func TestAddMetricsOutsideCurrentPeriod(t *testing.T) { Filter: Filter{ NamePass: []string{"*"}, }, - Period: time.Millisecond * 500, + DropOriginal: true, + Period: time.Millisecond * 500, }) require.NoError(t, ra.Config.Filter.Compile()) - acc := testutil.Accumulator{} + acc := &testutil.Accumulator{} now := time.Now() ra.UpdateWindow(now, now.Add(ra.Config.Period)) @@ -60,7 +98,8 @@ func TestAddMetricsOutsideCurrentPeriod(t *testing.T) { now.Add(-time.Hour), telegraf.Untyped, ) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) // metric after current period m = testutil.MustMetric("RITest", @@ -71,7 +110,8 @@ func TestAddMetricsOutsideCurrentPeriod(t *testing.T) { now.Add(time.Hour), telegraf.Untyped, ) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) // "now" metric m = testutil.MustMetric("RITest", @@ -81,9 +121,10 @@ func TestAddMetricsOutsideCurrentPeriod(t *testing.T) { }, time.Now().Add(time.Millisecond*50), telegraf.Untyped) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) - ra.Push(&acc) + ra.Push(acc) require.Equal(t, 1, len(acc.Metrics)) require.Equal(t, int64(101), acc.Metrics[0].Fields["sum"]) } @@ -95,11 +136,12 @@ func TestAddMetricsOutsideCurrentPeriodWithGrace(t *testing.T) { Filter: Filter{ NamePass: []string{"*"}, }, - Period: time.Millisecond * 1500, - Grace: time.Millisecond * 500, + Period: time.Millisecond * 1500, + Grace: time.Millisecond * 500, + DropOriginal: true, }) require.NoError(t, ra.Config.Filter.Compile()) - acc := testutil.Accumulator{} + acc := &testutil.Accumulator{} now := time.Now() ra.UpdateWindow(now, now.Add(ra.Config.Period)) @@ -111,7 +153,8 @@ func TestAddMetricsOutsideCurrentPeriodWithGrace(t *testing.T) { now.Add(-time.Hour), telegraf.Untyped, ) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) // metric before current period (late) m = testutil.MustMetric("RITest", @@ -122,7 +165,8 @@ func TestAddMetricsOutsideCurrentPeriodWithGrace(t *testing.T) { now.Add(-time.Millisecond*1000), telegraf.Untyped, ) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) // metric before current period, but within grace period (late) m = testutil.MustMetric("RITest", @@ -133,7 +177,8 @@ func TestAddMetricsOutsideCurrentPeriodWithGrace(t *testing.T) { now.Add(-time.Millisecond*200), telegraf.Untyped, ) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) // "now" metric m = testutil.MustMetric("RITest", @@ -143,9 +188,10 @@ func TestAddMetricsOutsideCurrentPeriodWithGrace(t *testing.T) { }, time.Now().Add(time.Millisecond*50), telegraf.Untyped) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) - ra.Push(&acc) + ra.Push(acc) require.Equal(t, 1, len(acc.Metrics)) require.Equal(t, int64(203), acc.Metrics[0].Fields["sum"]) } @@ -160,7 +206,7 @@ func TestAddAndPushOnePeriod(t *testing.T) { Period: time.Millisecond * 500, }) require.NoError(t, ra.Config.Filter.Compile()) - acc := testutil.Accumulator{} + acc := &testutil.Accumulator{} now := time.Now() ra.UpdateWindow(now, now.Add(ra.Config.Period)) @@ -172,9 +218,10 @@ func TestAddAndPushOnePeriod(t *testing.T) { }, time.Now().Add(time.Millisecond*100), telegraf.Untyped) - require.False(t, ra.Add(m)) + require.NoError(t, ra.Add(m, acc)) + require.Len(t, m.FieldList(), 1) - ra.Push(&acc) + ra.Push(acc) acc.AssertContainsFields(t, "TestMetric", map[string]interface{}{"sum": int64(101)}) } @@ -199,7 +246,8 @@ func TestAddDropOriginal(t *testing.T) { }, now, telegraf.Untyped) - require.True(t, ra.Add(m)) + require.NoError(t, ra.Add(m, &testutil.Accumulator{})) + require.Len(t, m.FieldList(), 1) // this metric name doesn't match the filter, so Add will return false m2 := testutil.MustMetric("foobar", @@ -209,7 +257,8 @@ func TestAddDropOriginal(t *testing.T) { }, now, telegraf.Untyped) - require.False(t, ra.Add(m2)) + require.NoError(t, ra.Add(m2, &testutil.Accumulator{})) + require.Len(t, m.FieldList(), 1) } func TestAddDoesNotModifyMetric(t *testing.T) { @@ -233,7 +282,8 @@ func TestAddDoesNotModifyMetric(t *testing.T) { }, now) expected := m.Copy() - ra.Add(m) + err := ra.Add(m, &testutil.Accumulator{}) + require.NoError(t, err) testutil.RequireMetricEqual(t, expected, m) } diff --git a/models/running_input.go b/models/running_input.go index 70a4c2ee3a70f..e7a5affa2ae95 100644 --- a/models/running_input.go +++ b/models/running_input.go @@ -13,6 +13,7 @@ var ( ) type RunningInput struct { + ID uint64 Input telegraf.Input Config *InputConfig @@ -21,6 +22,7 @@ type RunningInput struct { MetricsGathered selfstat.Stat GatherTime selfstat.Stat + State } func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { @@ -38,6 +40,7 @@ func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { SetLoggerOnPlugin(input, logger) return &RunningInput{ + ID: NextPluginID(), Input: input, Config: config, MetricsGathered: selfstat.Register( @@ -56,23 +59,58 @@ func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { // InputConfig is the common config for all inputs. type InputConfig struct { - Name string - Alias string - Interval time.Duration - CollectionJitter time.Duration - Precision time.Duration - - NameOverride string - MeasurementPrefix string - MeasurementSuffix string - Tags map[string]string - Filter Filter + Name string `toml:"name" json:"name"` + Alias string `toml:"alias" json:"alias"` + Interval time.Duration `toml:"interval" json:"interval"` + CollectionJitter time.Duration `toml:"collection_jitter" json:"collection_jitter"` + Precision time.Duration `toml:"precision" json:"precision"` + + NameOverride string `toml:"name_override" json:"name_override"` + MeasurementPrefix string `toml:"measurement_prefix" json:"measurement_prefix"` + MeasurementSuffix string `toml:"measurement_suffix" json:"measurement_suffix"` + Tags map[string]string `toml:"tags" json:"tags"` + Filter Filter `toml:"filter" json:"filter"` } func (r *RunningInput) metricFiltered(metric telegraf.Metric) { metric.Drop() } +func (r *RunningInput) Start(acc telegraf.Accumulator) error { + r.setState(PluginStateStarting) + if si, ok := r.Input.(telegraf.ServiceInput); ok { + if err := si.Start(acc); err != nil { + return err + } + } + r.setState(PluginStateRunning) + return nil +} + +func (r *RunningInput) Stop() { + state := r.GetState() + + switch state { + case PluginStateCreated, PluginStateStarting: + // shutting down before it got started. nothing to close + r.setState(PluginStateDead) + return + } + + if state != PluginStateRunning { + panic("plugin state was not running, it was: " + state.String()) + } + r.setState(PluginStateStopping) + if si, ok := r.Input.(telegraf.ServiceInput); ok { + si.Stop() + } + r.setState(PluginStateDead) +} + +func (r *RunningInput) GetID() uint64 { + return r.ID +} + func (r *RunningInput) LogName() string { return logName("inputs", r.Config.Name, r.Config.Alias) } diff --git a/models/running_output.go b/models/running_output.go index 6f5f8c0a84bad..e879a93797eaf 100644 --- a/models/running_output.go +++ b/models/running_output.go @@ -35,6 +35,8 @@ type OutputConfig struct { // RunningOutput contains the output configuration type RunningOutput struct { + ID uint64 + // Must be 64-bit aligned newMetricsCount int64 droppedMetrics int64 @@ -53,6 +55,7 @@ type RunningOutput struct { log telegraf.Logger aggMutex sync.Mutex + State } func NewRunningOutput( @@ -87,6 +90,7 @@ func NewRunningOutput( } ro := &RunningOutput{ + ID: NextPluginID(), buffer: NewBuffer(config.Name, config.Alias, bufferLimit), BatchReady: make(chan time.Time, 1), Output: output, @@ -105,6 +109,7 @@ func NewRunningOutput( ), log: logger, } + ro.setState(PluginStateCreated) return ro } @@ -143,13 +148,6 @@ func (r *RunningOutput) AddMetric(metric telegraf.Metric) { return } - if output, ok := r.Output.(telegraf.AggregatingOutput); ok { - r.aggMutex.Lock() - output.Add(metric) - r.aggMutex.Unlock() - return - } - if len(r.Config.NameOverride) > 0 { metric.SetName(r.Config.NameOverride) } @@ -178,14 +176,6 @@ func (r *RunningOutput) AddMetric(metric telegraf.Metric) { // Write writes all metrics to the output, stopping when all have been sent on // or error. func (r *RunningOutput) Write() error { - if output, ok := r.Output.(telegraf.AggregatingOutput); ok { - r.aggMutex.Lock() - metrics := output.Push() - r.buffer.Add(metrics...) - output.Reset() - r.aggMutex.Unlock() - } - atomic.StoreInt64(&r.newMetricsCount, 0) // Only process the metrics in the buffer now. Metrics added while we are @@ -227,10 +217,23 @@ func (r *RunningOutput) WriteBatch() error { // Close closes the output func (r *RunningOutput) Close() { + state := r.GetState() + + switch state { + case PluginStateCreated, PluginStateStarting: + r.setState(PluginStateDead) + return + } + + if state != PluginStateRunning { + panic("expected output plugin to be running, but was: " + state.String()) + } + r.setState(PluginStateStopping) err := r.Output.Close() if err != nil { r.log.Errorf("Error closing output: %v", err) } + r.setState(PluginStateDead) } func (r *RunningOutput) write(metrics []telegraf.Metric) error { @@ -251,6 +254,20 @@ func (r *RunningOutput) write(metrics []telegraf.Metric) error { return err } +func (r *RunningOutput) Connect() error { + r.setState(PluginStateStarting) + err := r.Output.Connect() + if err != nil { + return err + } + r.setState(PluginStateRunning) + return nil +} + +func (r *RunningOutput) GetID() uint64 { + return r.ID +} + func (r *RunningOutput) LogBufferStatus() { nBuffer := r.buffer.Len() r.log.Debugf("Buffer fullness: %d / %d metrics", nBuffer, r.MetricBufferLimit) diff --git a/models/running_plugin.go b/models/running_plugin.go new file mode 100644 index 0000000000000..226cccf0ea899 --- /dev/null +++ b/models/running_plugin.go @@ -0,0 +1,38 @@ +package models + +import ( + "strconv" + + "github.com/influxdata/telegraf" +) + +// PluginID is the random id assigned to the plugin so it can be referenced later +type PluginID string + +func (id PluginID) Uint64() uint64 { + result, _ := strconv.ParseUint(string(id), 16, 64) + return result +} + +type RunningPlugin interface { + Init() error + GetID() uint64 + GetState() PluginState +} + +// ProcessorRunner is an interface common to processors and aggregators so that aggregators can act like processors, including being ordered with and between processors +type ProcessorRunner interface { + RunningPlugin + Start(acc telegraf.Accumulator) error + Add(m telegraf.Metric, acc telegraf.Accumulator) error + Stop() + Order() int64 + LogName() string +} + +// ProcessorRunners add sorting +type ProcessorRunners []ProcessorRunner + +func (rp ProcessorRunners) Len() int { return len(rp) } +func (rp ProcessorRunners) Swap(i, j int) { rp[i], rp[j] = rp[j], rp[i] } +func (rp ProcessorRunners) Less(i, j int) bool { return rp[i].Order() < rp[j].Order() } diff --git a/models/running_processor.go b/models/running_processor.go index 5201fb27f19c0..8d56f6f3beacf 100644 --- a/models/running_processor.go +++ b/models/running_processor.go @@ -8,18 +8,14 @@ import ( ) type RunningProcessor struct { + ID uint64 sync.Mutex log telegraf.Logger Processor telegraf.StreamingProcessor Config *ProcessorConfig + State } -type RunningProcessors []*RunningProcessor - -func (rp RunningProcessors) Len() int { return len(rp) } -func (rp RunningProcessors) Swap(i, j int) { rp[i], rp[j] = rp[j], rp[i] } -func (rp RunningProcessors) Less(i, j int) bool { return rp[i].Config.Order < rp[j].Config.Order } - // FilterConfig containing a name and filter type ProcessorConfig struct { Name string @@ -41,14 +37,18 @@ func NewRunningProcessor(processor telegraf.StreamingProcessor, config *Processo }) SetLoggerOnPlugin(processor, logger) - return &RunningProcessor{ + p := &RunningProcessor{ + ID: NextPluginID(), Processor: processor, Config: config, log: logger, } + p.setState(PluginStateCreated) + return p } func (rp *RunningProcessor) metricFiltered(metric telegraf.Metric) { + //TODO(steve): rp.MetricsFiltered.Incr(1) metric.Drop() } @@ -66,6 +66,7 @@ func (rp *RunningProcessor) Log() telegraf.Logger { return rp.log } +// LogName returns the name of the processor as the logs would see it, with any alias. func (rp *RunningProcessor) LogName() string { return logName("processors", rp.Config.Name, rp.Config.Alias) } @@ -75,7 +76,13 @@ func (rp *RunningProcessor) MakeMetric(metric telegraf.Metric) telegraf.Metric { } func (rp *RunningProcessor) Start(acc telegraf.Accumulator) error { - return rp.Processor.Start(acc) + rp.setState(PluginStateStarting) + err := rp.Processor.Start(acc) + if err != nil { + return err + } + rp.setState(PluginStateRunning) + return nil } func (rp *RunningProcessor) Add(m telegraf.Metric, acc telegraf.Accumulator) error { @@ -96,5 +103,15 @@ func (rp *RunningProcessor) Add(m telegraf.Metric, acc telegraf.Accumulator) err } func (rp *RunningProcessor) Stop() { + rp.setState(PluginStateStopping) rp.Processor.Stop() + rp.setState(PluginStateDead) +} + +func (rp *RunningProcessor) Order() int64 { + return rp.Config.Order +} + +func (rp *RunningProcessor) GetID() uint64 { + return rp.ID } diff --git a/models/running_processor_test.go b/models/running_processor_test.go index 14df03253bd38..4116a025361ed 100644 --- a/models/running_processor_test.go +++ b/models/running_processor_test.go @@ -221,9 +221,9 @@ func TestRunningProcessor_Order(t *testing.T) { }, } - procs := models.RunningProcessors{rp2, rp3, rp1} + procs := models.ProcessorRunners{rp2, rp3, rp1} sort.Sort(procs) - require.Equal(t, - models.RunningProcessors{rp1, rp2, rp3}, + require.EqualValues(t, + models.ProcessorRunners{rp1, rp2, rp3}, procs) } diff --git a/models/state.go b/models/state.go new file mode 100644 index 0000000000000..7cd25ea5628f8 --- /dev/null +++ b/models/state.go @@ -0,0 +1,42 @@ +package models + +import "sync/atomic" + +// PluginState describes what the instantiated plugin is currently doing +// needs to stay int32 for use with atomic +type PluginState int32 + +const ( + PluginStateCreated PluginState = iota + PluginStateStarting + PluginStateRunning + PluginStateStopping + PluginStateDead +) + +func (p PluginState) String() string { + switch p { + case PluginStateCreated: + return "created" + case PluginStateStarting: + return "starting" + case PluginStateRunning: + return "running" + case PluginStateStopping: + return "stopping" + default: + return "dead" + } +} + +type State struct { + state PluginState +} + +func (s *State) setState(newState PluginState) { + atomic.StoreInt32((*int32)(&s.state), int32(newState)) +} + +func (s *State) GetState() PluginState { + return PluginState(atomic.LoadInt32((*int32)(&s.state))) +} diff --git a/output.go b/output.go index 52755b5da3f96..1ff4dfd2f7a7e 100644 --- a/output.go +++ b/output.go @@ -13,17 +13,3 @@ type Output interface { // Write takes in group of points to be written to the Output Write(metrics []Metric) error } - -// AggregatingOutput adds aggregating functionality to an Output. May be used -// if the Output only accepts a fixed set of aggregations over a time period. -// These functions may be called concurrently to the Write function. -type AggregatingOutput interface { - Output - - // Add the metric to the aggregator - Add(in Metric) - // Push returns the aggregated metrics and is called every flush interval. - Push() []Metric - // Reset signals the the aggregator period is completed. - Reset() -} diff --git a/plugin.go b/plugin.go index f9dcaeac0344c..73d10def431ff 100644 --- a/plugin.go +++ b/plugin.go @@ -16,10 +16,10 @@ type Initializer interface { // not part of the interface, but will receive an injected logger if it's set. // eg: Log telegraf.Logger `toml:"-"` type PluginDescriber interface { - // SampleConfig returns the default configuration of the Processor + // SampleConfig returns the default configuration of the Plugin SampleConfig() string - // Description returns a one-sentence description on the Processor + // Description returns a one-sentence description on the Plugin Description() string } diff --git a/plugins/configs/all/all.go b/plugins/configs/all/all.go new file mode 100644 index 0000000000000..3376811f52633 --- /dev/null +++ b/plugins/configs/all/all.go @@ -0,0 +1,6 @@ +package all + +import ( + // Blank imports for plugins to register themselves + _ "github.com/influxdata/telegraf/plugins/configs/api" +) diff --git a/plugins/configs/api/README.md b/plugins/configs/api/README.md new file mode 100644 index 0000000000000..e081fbdbfac41 --- /dev/null +++ b/plugins/configs/api/README.md @@ -0,0 +1,211 @@ +# Config API + +The Config API allows you to use HTTP requests to make changes to the set of running +plugins, starting or stopping plugins as needed without restarting Telegraf. When +configured with storage, the current list of running plugins and their configuration is +saved, and will persist across restarts of Telegraf. + +## Example Config + +```toml + [[config.api]] + service_address = ":7551" + [config.api.storage.internal] + file = "config_state.db" +``` + + +## Endpoints + +### GET /plugins/list + +List all known plugins with default config. Each plugin is listed once. + +**request params** + +None + +**response** + +An array of plugin-config schemas. + +eg: +```json +[ + { + "name": "mqtt_consumer", + "config": { + "servers": { + "type": "[string]", // another example: Map[string, SomeSchema] + "default": ["http://127.0.0.1"], + "required": true, + }, + "topics": { + "type": "[string]", + "default": [ + "telegraf/host01/cpu", + "telegraf/+/mem", + "sensors/#", + ], + "required": true, + }, + "topic_tag": { + "type": "string", + "default": "topic", + }, + "username": { + "type": "string", + "required": false, + }, + "password": { + "type": "string", + "required": false, + }, + "qos": { + "type": "integer", + "format": "int64", + }, + "connection_timeout": { + "type": "integer", + "format": "duration" + }, + "max_undelivered_messages": { + "type": "integer", + "format": "int32", + } + } + }, + // ... +] +``` + +### GET /plugins/running + +List all currently running plugins. If there are 5 copies of a plugin, all 5 will be returned. + +**request params** + +none + +**response** + +```json +[ + { + "id": "unique-id-here", + "name": "mqtt_consumer", + "config": { + "servers": ["tcp://127.0.0.1:1883"], + "topics": [ + "telegraf/host01/cpu", + "telegraf/+/mem", + "sensors/#", + ], + "topic_tag": "topic", + "qos": 0, + "connection_timeout": 300000000000, + "max_undelivered_messages": 1000, + "persistent_session": false, + "client_id": "", + "username": "telegraf", + // some fields, like "password", are settable, but their values are not returned + "password": "********", + "tls_ca": "/etc/telegraf/ca.pem", + "tls_cert": "/etc/telegraf/cert.pem", + "tls_key": "/etc/telegraf/key.pem", + "insecure_skip_verify": false, + "data_format": "influx", + }, + }, +] +``` + +### POST /plugins/create + +Create a new plugin. It will be started upon creation. + +**request params** + +```json + { + "name": "inputs.mqtt_consumer", + "config": { + // .. + }, + }, +``` + +**response** + +```json + {"id": "unique-id-here"} +``` + +### GET /plugins/{id}/status + +Get the status of a launched plugin + +**request params** + +None. ID in url + +**response** + +```json + { + "status": "", // starting, running, notfound, or error + "reason": "", // extended reason code containing error details. + } +``` + +### DELETE /plugins/{id} + +Stop an existing running plugin given its `id`. It will be allowed to finish +any metrics in-progress. + +**request params** + +None + +**response** + +200 OK +```json +{} +``` + +## Schemas + +### plugin-config + +A plugin-config is a plugin name and details about the config fields. + +``` + { + name: string, + config: Map[string, FieldConfig] + } +``` + +### FieldConfig + +``` + { + type: string, // eg "string", "integer", "[string]", or "Map[string, SomeSchema]" + default: object, // whatever the default value is + required: bool, + format: string, // type-specific format info. + } +``` + +### plugin + +An instance of a plugin running with a specific configuration + +``` +{ + id: string, + name: string, + config: Map[string, object], +} +``` diff --git a/plugins/configs/api/controller.go b/plugins/configs/api/controller.go new file mode 100644 index 0000000000000..3c3693807bfcd --- /dev/null +++ b/plugins/configs/api/controller.go @@ -0,0 +1,1188 @@ +package api + +import ( + "context" + "errors" + "fmt" + "log" // nolint:revive + "reflect" + "regexp" + "sort" + "strings" + "time" + "unicode" + + "github.com/alecthomas/units" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" + "github.com/influxdata/telegraf/plugins/aggregators" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/plugins/outputs" + "github.com/influxdata/telegraf/plugins/parsers" + "github.com/influxdata/telegraf/plugins/processors" + "github.com/influxdata/telegraf/plugins/serializers" +) + +// api is the general interface to interacting with Telegraf's current config +type api struct { + agent config.AgentController + config *config.Config + + // api shutdown context + ctx context.Context + outputCtx context.Context + + addHooks []PluginCallbackEvent + removeHooks []PluginCallbackEvent +} + +// nolint:revive +func newAPI(ctx context.Context, outputCtx context.Context, cfg *config.Config, agent config.AgentController) *api { + c := &api{ + config: cfg, + agent: agent, + ctx: ctx, + outputCtx: outputCtx, + } + return c +} + +// PluginConfigTypeInfo is a plugin name and details about the config fields. +type PluginConfigTypeInfo struct { + Name string `json:"name"` + Config map[string]FieldConfig `json:"config"` +} + +type PluginConfig struct { + ID string `json:"id"` // unique identifer + PluginConfigCreate +} + +type PluginConfigCreate struct { + Name string `json:"name"` // name of the plugin + Config map[string]interface{} `json:"config"` // map field name to field value +} + +// FieldConfig describes a single field +type FieldConfig struct { + Type FieldType `json:"type,omitempty"` // see FieldType + Default interface{} `json:"default,omitempty"` // whatever the default value is + Format string `json:"format,omitempty"` // type-specific format info. eg a url is a string, but has url-formatting rules. + Required bool `json:"required,omitempty"` // this is sort of validation, which I'm not sure belongs here. + SubType FieldType `json:"sub_type,omitempty"` // The subtype. map[string]int subtype is int. []string subtype is string. + SubFields map[string]FieldConfig `json:"sub_fields,omitempty"` // only for struct/object/FieldConfig types +} + +// FieldType enumerable type. Describes config field type information to external applications +type FieldType string + +// FieldTypes +const ( + FieldTypeUnknown FieldType = "" + FieldTypeString FieldType = "string" + FieldTypeInteger FieldType = "integer" + FieldTypeDuration FieldType = "duration" // a special case of integer + FieldTypeSize FieldType = "size" // a special case of integer + FieldTypeFloat FieldType = "float" + FieldTypeBool FieldType = "bool" + FieldTypeInterface FieldType = "any" + FieldTypeSlice FieldType = "array" // array + FieldTypeFieldConfig FieldType = "object" // a FieldConfig? + FieldTypeMap FieldType = "map" // always map[string]FieldConfig ? +) + +// Plugin is an instance of a plugin running with a specific configuration +type Plugin struct { + ID models.PluginID + Name string + // State() + Config map[string]interface{} +} + +func (a *api) ListPluginTypes() []PluginConfigTypeInfo { + result := []PluginConfigTypeInfo{} + inputNames := []string{} + for name := range inputs.Inputs { + inputNames = append(inputNames, name) + } + sort.Strings(inputNames) + + for _, name := range inputNames { + creator := inputs.Inputs[name] + cfg := PluginConfigTypeInfo{ + Name: "inputs." + name, + Config: map[string]FieldConfig{}, + } + + p := creator() + getFieldConfig(p, cfg.Config) + + result = append(result, cfg) + } + + processorNames := []string{} + for name := range processors.Processors { + processorNames = append(processorNames, name) + } + sort.Strings(processorNames) + + for _, name := range processorNames { + creator := processors.Processors[name] + cfg := PluginConfigTypeInfo{ + Name: "processors." + name, + Config: map[string]FieldConfig{}, + } + + p := creator() + getFieldConfig(p, cfg.Config) + + result = append(result, cfg) + } + + aggregatorNames := []string{} + for name := range aggregators.Aggregators { + aggregatorNames = append(aggregatorNames, name) + } + sort.Strings(aggregatorNames) + + for _, name := range aggregatorNames { + creator := aggregators.Aggregators[name] + cfg := PluginConfigTypeInfo{ + Name: "aggregators." + name, + Config: map[string]FieldConfig{}, + } + + p := creator() + getFieldConfig(p, cfg.Config) + + result = append(result, cfg) + } + + outputNames := []string{} + for name := range outputs.Outputs { + outputNames = append(outputNames, name) + } + sort.Strings(outputNames) + + for _, name := range outputNames { + creator := outputs.Outputs[name] + cfg := PluginConfigTypeInfo{ + Name: "outputs." + name, + Config: map[string]FieldConfig{}, + } + + p := creator() + getFieldConfig(p, cfg.Config) + + result = append(result, cfg) + } + + return result +} + +func (a *api) ListRunningPlugins() (runningPlugins []Plugin) { + if a == nil { + panic("api is nil") + } + for _, v := range a.agent.RunningInputs() { + p := Plugin{ + ID: idToString(v.ID), + Name: v.LogName(), + Config: map[string]interface{}{}, + } + getFieldConfigValuesFromStruct(v.Config, p.Config) + getFieldConfigValuesFromStruct(v.Input, p.Config) + runningPlugins = append(runningPlugins, p) + } + for _, v := range a.agent.RunningProcessors() { + p := Plugin{ + ID: idToString(v.GetID()), + Name: v.LogName(), + Config: map[string]interface{}{}, + } + val := reflect.ValueOf(v) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + pluginCfg := val.FieldByName("Config").Interface() + getFieldConfigValuesFromStruct(pluginCfg, p.Config) + if proc := val.FieldByName("Processor"); proc.IsValid() && !proc.IsNil() { + getFieldConfigValuesFromStruct(proc.Interface(), p.Config) + } + if agg := val.FieldByName("Aggregator"); agg.IsValid() && !agg.IsNil() { + getFieldConfigValuesFromStruct(agg.Interface(), p.Config) + } + runningPlugins = append(runningPlugins, p) + } + for _, v := range a.agent.RunningOutputs() { + p := Plugin{ + ID: idToString(v.ID), + Name: v.LogName(), + Config: map[string]interface{}{}, + } + getFieldConfigValuesFromStruct(v.Config, p.Config) + getFieldConfigValuesFromStruct(v.Output, p.Config) + runningPlugins = append(runningPlugins, p) + } + + if runningPlugins == nil { + return []Plugin{} + } + return runningPlugins +} + +func (a *api) UpdatePlugin(id models.PluginID, cfg PluginConfigCreate) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + // TODO: shut down plugin and start a new plugin with the same id. + if err := a.DeletePlugin(id); err != nil { + return err + } + // wait for plugin to stop before recreating it with the same ID, otherwise we'll have issues. + for a.GetPluginStatus(id) != models.PluginStateDead { + select { + case <-ctx.Done(): + // plugin didn't stop in time.. do we recreate it anyway? + return errors.New("timed out shutting down plugin for update") + case <-time.After(100 * time.Millisecond): + // try again + } + } + _, err := a.CreatePlugin(cfg, id) + return err +} + +// CreatePlugin creates a new plugin from a specified config. forcedID should be left blank when used by users via the API. +func (a *api) CreatePlugin(cfg PluginConfigCreate, forcedID models.PluginID) (models.PluginID, error) { + log.Printf("I! [configapi] creating plugin %q", cfg.Name) + + parts := strings.Split(cfg.Name, ".") + pluginType, name := parts[0], parts[1] + switch pluginType { + case "inputs": + // add an input + input, ok := inputs.Inputs[name] + if !ok { + return "", fmt.Errorf("%w: finding plugin with name %s", ErrNotFound, name) + } + // create a copy + i := input() + // set the config + if err := setFieldConfig(cfg.Config, i); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + + // get parser! + if t, ok := i.(parsers.ParserInput); ok { + pc := &parsers.Config{ + MetricName: name, + JSONStrict: true, + DataFormat: "influx", + } + if err := setFieldConfig(cfg.Config, pc); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + parser, err := parsers.NewParser(pc) + if err != nil { + return "", fmt.Errorf("%w: setting parser %s", ErrBadRequest, err) + } + t.SetParser(parser) + } + + if t, ok := i.(parsers.ParserFuncInput); ok { + pc := &parsers.Config{ + MetricName: name, + JSONStrict: true, + DataFormat: "influx", + } + if err := setFieldConfig(cfg.Config, pc); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + + t.SetParserFunc(func() (parsers.Parser, error) { + return parsers.NewParser(pc) + }) + } + + // start it and put it into the agent manager? + pluginConfig := &models.InputConfig{Name: name} + if err := setFieldConfig(cfg.Config, pluginConfig); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + + if err := setFieldConfig(cfg.Config, &pluginConfig.Filter); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + + ri := models.NewRunningInput(i, pluginConfig) + if len(forcedID) > 0 { + ri.ID = forcedID.Uint64() + } + ri.SetDefaultTags(a.config.Tags) + + if err := ri.Init(); err != nil { + return "", fmt.Errorf("%w: could not initialize plugin %s", ErrBadRequest, err) + } + + a.agent.AddInput(ri) + a.addPluginHook(PluginConfig{ID: string(idToString(ri.ID)), PluginConfigCreate: PluginConfigCreate{ + Name: "inputs." + name, // TODO: use PluginName() or something + Config: cfg.Config, + }}) + + go a.agent.RunInput(ri, time.Now()) + + return idToString(ri.ID), nil + case "outputs": + // add an output + output, ok := outputs.Outputs[name] + if !ok { + return "", fmt.Errorf("%w: Error finding plugin with name %s", ErrNotFound, name) + } + // create a copy + o := output() + // set the config + if err := setFieldConfig(cfg.Config, o); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + // start it and put it into the agent manager? + pluginConfig := &models.OutputConfig{Name: name} + if err := setFieldConfig(cfg.Config, pluginConfig); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + + if err := setFieldConfig(cfg.Config, &pluginConfig.Filter); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + + if t, ok := o.(serializers.SerializerOutput); ok { + sc := &serializers.Config{ + TimestampUnits: 1 * time.Second, + DataFormat: "influx", + } + if err := setFieldConfig(cfg.Config, sc); err != nil { + return "", fmt.Errorf("%w: setting field %s", ErrBadRequest, err) + } + serializer, err := serializers.NewSerializer(sc) + if err != nil { + return "", fmt.Errorf("%w: setting serializer %s", ErrBadRequest, err) + } + t.SetSerializer(serializer) + } + + ro := models.NewRunningOutput(o, pluginConfig, a.config.Agent.MetricBatchSize, a.config.Agent.MetricBufferLimit) + + if err := ro.Init(); err != nil { + return "", fmt.Errorf("%w: could not initialize plugin %s", ErrBadRequest, err) + } + + a.agent.AddOutput(ro) + + a.addPluginHook(PluginConfig{ID: string(idToString(ro.ID)), PluginConfigCreate: PluginConfigCreate{ + Name: "outputs." + name, // TODO: use PluginName() or something + Config: cfg.Config, + }}) + + go a.agent.RunOutput(a.outputCtx, ro) + + return idToString(ro.ID), nil + case "aggregators": + aggregator, ok := aggregators.Aggregators[name] + if !ok { + return "", fmt.Errorf("%w: Error finding aggregator plugin with name %s", ErrNotFound, name) + } + agg := aggregator() + + // set the config + if err := setFieldConfig(cfg.Config, agg); err != nil { + return "", fmt.Errorf("%w: setting field", ErrBadRequest) + } + aggCfg := &models.AggregatorConfig{ + Name: name, + Delay: time.Millisecond * 100, + Period: time.Second * 30, + Grace: time.Second * 0, + } + if err := setFieldConfig(cfg.Config, aggCfg); err != nil { + return "", fmt.Errorf("%w: setting field", ErrBadRequest) + } + + if err := setFieldConfig(cfg.Config, &aggCfg.Filter); err != nil { + return "", fmt.Errorf("%w: setting field", ErrBadRequest) + } + + ra := models.NewRunningAggregator(agg, aggCfg) + if err := ra.Init(); err != nil { + return "", fmt.Errorf("%w: could not initialize plugin %s", ErrBadRequest, err) + } + a.agent.AddProcessor(ra) + + a.addPluginHook(PluginConfig{ID: string(idToString(ra.ID)), PluginConfigCreate: PluginConfigCreate{ + Name: "aggregators." + name, // TODO: use PluginName() or something + Config: cfg.Config, + }}) + + go a.agent.RunProcessor(ra) + + return idToString(ra.ID), nil + + case "processors": + processor, ok := processors.Processors[name] + if !ok { + return "", fmt.Errorf("%w: Error finding processor plugin with name %s", ErrNotFound, name) + } + // create a copy + p := processor() + rootp := p.(telegraf.PluginDescriber) + if unwrapme, ok := rootp.(unwrappable); ok { + rootp = unwrapme.Unwrap() + } + // set the config + if err := setFieldConfig(cfg.Config, rootp); err != nil { + return "", fmt.Errorf("%w: setting field", ErrBadRequest) + } + // start it and put it into the agent manager? + pluginConfig := &models.ProcessorConfig{Name: name} + if err := setFieldConfig(cfg.Config, pluginConfig); err != nil { + return "", fmt.Errorf("%w: setting field", ErrBadRequest) + } + if err := setFieldConfig(cfg.Config, &pluginConfig.Filter); err != nil { + return "", fmt.Errorf("%w: setting field", ErrBadRequest) + } + + rp := models.NewRunningProcessor(p, pluginConfig) + + if err := rp.Init(); err != nil { + return "", fmt.Errorf("%w: could not initialize plugin %s", ErrBadRequest, err) + } + + a.agent.AddProcessor(rp) + + a.addPluginHook(PluginConfig{ID: string(idToString(rp.ID)), PluginConfigCreate: PluginConfigCreate{ + Name: "processors." + name, // TODO: use PluginName() or something + Config: cfg.Config, + }}) + + go a.agent.RunProcessor(rp) + + return idToString(rp.ID), nil + default: + return "", fmt.Errorf("%w: Unknown plugin type", ErrNotFound) + } +} + +func (a *api) GetPluginStatus(id models.PluginID) models.PluginState { + for _, v := range a.agent.RunningInputs() { + if v.ID == id.Uint64() { + return v.GetState() + } + } + for _, v := range a.agent.RunningProcessors() { + if v.GetID() == id.Uint64() { + return v.GetState() + } + } + for _, v := range a.agent.RunningOutputs() { + if v.ID == id.Uint64() { + return v.GetState() + } + } + return models.PluginStateDead +} + +func (a *api) DeletePlugin(id models.PluginID) error { + a.removePluginHook(PluginConfig{ID: string(id)}) + + for _, v := range a.agent.RunningInputs() { + if v.ID == id.Uint64() { + log.Printf("I! [configapi] stopping plugin %q", v.LogName()) + a.agent.StopInput(v) + return nil + } + } + for _, v := range a.agent.RunningProcessors() { + if v.GetID() == id.Uint64() { + log.Printf("I! [configapi] stopping plugin %q", v.LogName()) + a.agent.StopProcessor(v) + return nil + } + } + for _, v := range a.agent.RunningOutputs() { + if v.ID == id.Uint64() { + log.Printf("I! [configapi] stopping plugin %q", v.LogName()) + a.agent.StopOutput(v) + return nil + } + } + return ErrNotFound +} + +type PluginCallbackEvent func(p PluginConfig) + +// addPluginHook triggers the hook to fire this event +func (a *api) addPluginHook(p PluginConfig) { + for _, h := range a.addHooks { + h(p) + } +} + +// removePluginHook triggers the hook to fire this event +func (a *api) removePluginHook(p PluginConfig) { + for _, h := range a.removeHooks { + h(p) + } +} + +// OnPluginAdded adds a hook to get notified of this event +func (a *api) OnPluginAdded(f PluginCallbackEvent) { + a.addHooks = append(a.addHooks, f) +} + +// OnPluginRemoved adds a hook to get notified of this event +func (a *api) OnPluginRemoved(f PluginCallbackEvent) { + a.removeHooks = append(a.removeHooks, f) +} + +// setFieldConfig takes a map of field names to field values and sets them on the plugin +func setFieldConfig(cfg map[string]interface{}, p interface{}) error { + destStruct := reflect.ValueOf(p) + if destStruct.Kind() == reflect.Ptr { + destStruct = destStruct.Elem() + } + keys := make([]string, 0, len(cfg)) + for k := range cfg { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + v := cfg[k] + destField, destFieldType := getFieldByName(destStruct, k) // get by tag + if !destField.IsValid() { + continue + } + if !destField.CanSet() { + destField.Addr() + return fmt.Errorf("cannot set %s (%s)", k, destFieldType.Name()) + } + val := reflect.ValueOf(v) + if err := setObject(val, destField, destFieldType); err != nil { + return fmt.Errorf("Could not set field %q: %w", k, err) + } + } + return nil +} + +// getFieldByName gets a reference to a struct field from it's name, considering the tag names +func getFieldByName(destStruct reflect.Value, fieldName string) (reflect.Value, reflect.Type) { + if destStruct.Kind() == reflect.Ptr { + if destStruct.IsNil() { + return reflect.ValueOf(nil), reflect.TypeOf(nil) + } + destStruct = destStruct.Elem() + } + // may be an interface to a struct + if destStruct.Kind() == reflect.Interface { + destStruct = destStruct.Elem() + } + destStructType := reflect.TypeOf(destStruct.Interface()) + for i := 0; i < destStruct.NumField(); i++ { + field := destStruct.Field(i) + fieldType := destStructType.Field(i) + if fieldType.Type.Kind() == reflect.Struct && fieldType.Anonymous { + v, t := getFieldByName(field, fieldName) + if t != reflect.TypeOf(nil) { + return v, t + } + } + if fieldType.Tag.Get("toml") == fieldName { + return field, fieldType.Type + } + if name, ok := toSnakeCase(fieldType.Name, fieldType); ok { + if name == fieldName && isExported(fieldType) { + return field, fieldType.Type + } + } + } + return reflect.ValueOf(nil), reflect.TypeOf(nil) +} + +// getFieldConfig builds FieldConfig objects based on the structure of a plugin's struct +// it expects a plugin, p, (of any type) and a map to populate. +// it calls itself recursively so p must be an interface{} +func getFieldConfig(p interface{}, cfg map[string]FieldConfig) { + structVal := reflect.ValueOf(p) + structType := structVal.Type() + for structType.Kind() == reflect.Ptr { + structVal = structVal.Elem() + structType = structType.Elem() + } + + // safety check. + if structType.Kind() != reflect.Struct { + // woah, what? + panic(fmt.Sprintf("getFieldConfig expected a struct type, but got %v %v", p, structType.String())) + } + // structType.NumField() + + for i := 0; i < structType.NumField(); i++ { + var f reflect.Value + if !structVal.IsZero() { + f = structVal.Field(i) + } + _ = f + ft := structType.Field(i) + + ftType := ft.Type + if ftType.Kind() == reflect.Ptr { + ftType = ftType.Elem() + // f = f.Elem() + } + + // check if it's not exported, and skip if so. + if len(ft.Name) > 0 && strings.ToLower(string(ft.Name[0])) == string(ft.Name[0]) { + continue + } + tomlTag := ft.Tag.Get("toml") + if tomlTag == "-" { + continue + } + switch ftType.Kind() { + case reflect.Func, reflect.Interface: + continue + } + + // if this field is a struct, get the structure of it. + if ftType.Kind() == reflect.Struct && !isInternalStructFieldType(ft.Type) { + if ft.Anonymous { // embedded + t := getSubTypeType(ft.Type) + i := reflect.New(t) + getFieldConfig(i.Interface(), cfg) + } else { + subCfg := map[string]FieldConfig{} + t := getSubTypeType(ft.Type) + i := reflect.New(t) + getFieldConfig(i.Interface(), subCfg) + cfg[ft.Name] = FieldConfig{ + Type: FieldTypeFieldConfig, + SubFields: subCfg, + SubType: getFieldType(t), + } + } + continue + } + + // all other field types... + fc := FieldConfig{ + Type: getFieldTypeFromStructField(ft), + Format: ft.Tag.Get("format"), + Required: ft.Tag.Get("required") == "true", + } + + // set the default value for the field + if f.IsValid() && !f.IsZero() { + fc.Default = f.Interface() + // special handling for internal struct types so the struct doesn't serialize to an object. + if d, ok := fc.Default.(config.Duration); ok { + fc.Default = d + } + if s, ok := fc.Default.(config.Size); ok { + fc.Default = s + } + } + + // if we found a slice of objects, get the structure of that object + if hasSubType(ft.Type) { + t := getSubTypeType(ft.Type) + n := t.Name() + _ = n + fc.SubType = getFieldType(t) + + if t.Kind() == reflect.Struct { + i := reflect.New(t) + subCfg := map[string]FieldConfig{} + getFieldConfig(i.Interface(), subCfg) + fc.SubFields = subCfg + } + } + // if we found a map of objects, get the structure of that object + + cfg[ft.Name] = fc + } +} + +// getFieldConfigValuesFromStruct takes a struct and populates a map. +func getFieldConfigValuesFromStruct(p interface{}, cfg map[string]interface{}) { + structVal := reflect.ValueOf(p) + structType := structVal.Type() + if structVal.IsZero() { + return + } + + for structType.Kind() == reflect.Ptr { + structVal = structVal.Elem() + structType = structType.Elem() + } + + // safety check. + if structType.Kind() != reflect.Struct { + // woah, what? + panic(fmt.Sprintf("getFieldConfigValues expected a struct type, but got %v %v", p, structType.String())) + } + + for i := 0; i < structType.NumField(); i++ { + f := structVal.Field(i) + ft := structType.Field(i) + + if !isExported(ft) { + continue + } + if ft.Name == "Log" { + continue + } + ftType := ft.Type + if ftType.Kind() == reflect.Ptr { + f = f.Elem() + } + // if struct call self recursively + // if it's composed call self recursively + if name, ok := toSnakeCase(ft.Name, ft); ok { + setMapKey(cfg, name, f) + } + } +} + +func getFieldConfigValuesFromValue(val reflect.Value) interface{} { + typ := val.Type() + // typ may be a pointer to a type + if typ.Kind() == reflect.Ptr { + val = val.Elem() + typ = val.Type() + } + + switch typ.Kind() { + case reflect.Slice: + return getFieldConfigValuesFromSlice(val) + case reflect.Struct: + m := map[string]interface{}{} + getFieldConfigValuesFromStruct(val.Interface(), m) + return m + case reflect.Map: + return getFieldConfigValuesFromMap(val) + case reflect.Bool: + return val.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + return val.Int() + case reflect.Int64: + // special case for config.Duration, time.Duration etc. + switch typ.String() { + case "time.Duration", "config.Duration": + return time.Duration(val.Int()).String() + case "config.Size": + sz := config.Size(val.Int()) + return string((&sz).MarshalText()) + } + return val.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return val.Uint() + case reflect.Float32, reflect.Float64: + return val.Float() + case reflect.Interface: + return val.Interface() // do we bother to decode this? + case reflect.Ptr: + return getFieldConfigValuesFromValue(val.Elem()) + case reflect.String: + return val.String() + default: + return val.Interface() // log that we missed the type? + } +} + +func getFieldConfigValuesFromMap(f reflect.Value) map[string]interface{} { + obj := map[string]interface{}{} + iter := f.MapRange() + for iter.Next() { + setMapKey(obj, iter.Key().String(), iter.Value()) + } + return obj +} + +func getFieldConfigValuesFromSlice(val reflect.Value) []interface{} { + s := []interface{}{} + for i := 0; i < val.Len(); i++ { + s = append(s, getFieldConfigValuesFromValue(val.Index(i))) + } + return s +} + +func setMapKey(obj map[string]interface{}, key string, v reflect.Value) { + v = reflect.ValueOf(getFieldConfigValuesFromValue(v)) + switch v.Kind() { + case reflect.Bool: + obj[key] = v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + obj[key] = v.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + obj[key] = v.Uint() + case reflect.Float32, reflect.Float64: + obj[key] = v.Float() + case reflect.Interface: + obj[key] = v.Interface() + case reflect.Map: + obj[key] = v.Interface().(map[string]interface{}) + case reflect.Ptr: + setMapKey(obj, key, v.Elem()) + case reflect.Slice: + obj[key] = v.Interface() + case reflect.String: + obj[key] = v.String() + case reflect.Struct: + obj[key] = v.Interface() + default: + // obj[key] = v.Interface() + panic("unhandled type " + v.Type().String() + " for field " + key) + } +} + +func isExported(ft reflect.StructField) bool { + return unicode.IsUpper(rune(ft.Name[0])) +} + +var matchFirstCapital = regexp.MustCompile("(.)([A-Z][a-z]+)") +var matchAllCapitals = regexp.MustCompile("([a-z0-9])([A-Z])") + +func toSnakeCase(str string, sf reflect.StructField) (result string, ok bool) { + if toml, ok := sf.Tag.Lookup("toml"); ok { + if toml == "-" { + return "", false + } + return toml, true + } + snakeStr := matchFirstCapital.ReplaceAllString(str, "${1}_${2}") + snakeStr = matchAllCapitals.ReplaceAllString(snakeStr, "${1}_${2}") + return strings.ToLower(snakeStr), true +} + +func setObject(from, to reflect.Value, destType reflect.Type) error { + if from.Kind() == reflect.Interface { + from = reflect.ValueOf(from.Interface()) + } + // switch on source type + switch from.Kind() { + case reflect.Bool: + if to.Kind() == reflect.Ptr { + ptr := reflect.New(destType.Elem()) + to.Set(ptr) + to = ptr.Elem() + } + if to.Kind() == reflect.Interface { + to.Set(from) + } else { + to.SetBool(from.Bool()) + } + case reflect.String: + if to.Kind() == reflect.Ptr { + ptr := reflect.New(destType.Elem()) + destType = destType.Elem() + to.Set(ptr) + to = ptr.Elem() + } + // consider duration and size types + switch destType.String() { + case "time.Duration", "config.Duration": + d, err := time.ParseDuration(from.Interface().(string)) + if err != nil { + return fmt.Errorf("Couldn't parse duration %q: %w", from.Interface().(string), err) + } + to.SetInt(int64(d)) + case "config.Size": + size, err := units.ParseStrictBytes(from.Interface().(string)) + if err != nil { + return fmt.Errorf("Couldn't parse size %q: %w", from.Interface().(string), err) + } + to.SetInt(size) + // TODO: handle slice types? + default: + // to.SetString(from.Interface().(string)) + to.Set(from) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if to.Kind() == reflect.Ptr { + ptr := reflect.New(destType.Elem()) + destType = destType.Elem() + to.Set(ptr) + to = ptr.Elem() + } + + if destType.String() == "internal.Number" { + n := float64(from.Int()) + to.Set(reflect.ValueOf(n)) + return nil + } + + switch destType.Kind() { + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + to.SetUint(uint64(from.Int())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + to.SetInt(from.Int()) + case reflect.Float32, reflect.Float64: + to.SetFloat(from.Float()) + case reflect.Interface: + to.Set(from) + default: + return fmt.Errorf("cannot coerce int type into %s", destType.Kind().String()) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if to.Kind() == reflect.Ptr { + ptr := reflect.New(destType.Elem()) + // destType = destType.Elem() + to.Set(ptr) + to = ptr.Elem() + } + + if destType.String() == "internal.Number" { + n := float64(from.Uint()) + to.Set(reflect.ValueOf(n)) + return nil + } + + if to.Kind() == reflect.Interface { + to.Set(from) + } else { + to.SetUint(from.Uint()) + } + case reflect.Float32, reflect.Float64: + if to.Kind() == reflect.Ptr { + ptr := reflect.New(destType.Elem()) + // destType = destType.Elem() + to.Set(ptr) + to = ptr.Elem() + } + if destType.String() == "internal.Number" { + n := from.Float() + to.Set(reflect.ValueOf(n)) + return nil + } + + if to.Kind() == reflect.Interface { + to.Set(from) + } else { + to.SetFloat(from.Float()) + } + case reflect.Slice: + if destType.Kind() == reflect.Ptr { + destType = destType.Elem() + to = to.Elem() + } + if destType.Kind() != reflect.Slice { + return fmt.Errorf("error setting slice field into %s", destType.Kind().String()) + } + d := reflect.MakeSlice(destType, from.Len(), from.Len()) + for i := 0; i < from.Len(); i++ { + if err := setObject(from.Index(i), d.Index(i), destType.Elem()); err != nil { + return fmt.Errorf("couldn't set slice element: %w", err) + } + } + to.Set(d) + case reflect.Map: + if destType.Kind() == reflect.Ptr { + destType = destType.Elem() + ptr := reflect.New(destType) + to.Set(ptr) + to = to.Elem() + } + switch destType.Kind() { + case reflect.Struct: + structPtr := reflect.New(destType) + err := setFieldConfig(from.Interface().(map[string]interface{}), structPtr.Interface()) + if err != nil { + return err + } + to.Set(structPtr.Elem()) + case reflect.Map: + //TODO: handle map[string]type + if destType.Key().Kind() != reflect.String { + panic("expecting string types for maps") + } + to.Set(reflect.MakeMap(destType)) + + switch destType.Elem().Kind() { + case reflect.Interface, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, + reflect.Bool: + for _, k := range from.MapKeys() { + t := from.MapIndex(k) + if t.Kind() == reflect.Interface { + t = reflect.ValueOf(t.Interface()) + } + to.SetMapIndex(k, t) + } + case reflect.String: + for _, k := range from.MapKeys() { + t := from.MapIndex(k) + if t.Kind() == reflect.Interface { + t = reflect.ValueOf(t.Interface()) + } + to.SetMapIndex(k, t) + } + // for _, k := range from.MapKeys() { + // v := from.MapIndex(k) + // s := v.Interface().(string) + // to.SetMapIndex(k, reflect.ValueOf(s)) + // } + case reflect.Slice: + for _, k := range from.MapKeys() { + // slice := reflect.MakeSlice(destType.Elem(), 0, 0) + sliceptr := reflect.New(destType.Elem()) + // sliceptr.Elem().Set(slice) + err := setObject(from.MapIndex(k), sliceptr, sliceptr.Type()) + if err != nil { + return fmt.Errorf("could not set slice: %w", err) + } + to.SetMapIndex(k, sliceptr.Elem()) + } + + case reflect.Struct: + for _, k := range from.MapKeys() { + structPtr := reflect.New(destType.Elem()) + err := setFieldConfig( + from.MapIndex(k).Interface().(map[string]interface{}), + structPtr.Interface(), + ) + // err := setObject(from.MapIndex(k), structPtr, structPtr.Type()) + if err != nil { + return fmt.Errorf("could not set struct: %w", err) + } + to.SetMapIndex(k, structPtr.Elem()) + } + + default: + return fmt.Errorf("can't write settings into map of type map[string]%s", destType.Elem().Kind().String()) + } + default: + return fmt.Errorf("Cannot load map into %q", destType.Kind().String()) + // panic("foo") + } + // to.Set(val) + default: + return fmt.Errorf("cannot convert unknown type %s to %s", from.Kind().String(), destType.String()) + } + return nil +} + +// hasSubType returns true when the field has a subtype (map,slice,struct) +func hasSubType(t reflect.Type) bool { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + switch t.Kind() { + case reflect.Slice, reflect.Map: + return true + case reflect.Struct: + switch t.String() { + case "internal.Duration", "config.Duration", "internal.Size", "config.Size": + return false + } + return true + default: + return false + } +} + +// getSubTypeType gets the underlying subtype's reflect.Type +// examples: +// []string => string +// map[string]int => int +// User => User +func getSubTypeType(typ reflect.Type) reflect.Type { + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + switch typ.Kind() { + case reflect.Slice: + t := typ.Elem() + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t + case reflect.Map: + t := typ.Elem() + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t + case reflect.Struct: + return typ + } + panic(typ.String() + " is not a type that has subtype information (map, slice, struct)") +} + +// getFieldType translates reflect.Types to our API field types. +func getFieldType(t reflect.Type) FieldType { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + switch t.Kind() { + case reflect.String: + return FieldTypeString + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, + reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, + reflect.Uint32, reflect.Uint64: + return FieldTypeInteger + case reflect.Float32, reflect.Float64: + return FieldTypeFloat + case reflect.Bool: + return FieldTypeBool + case reflect.Slice: + return FieldTypeSlice + case reflect.Map: + return FieldTypeMap + case reflect.Struct: + switch t.String() { + case "internal.Duration", "config.Duration": + return FieldTypeDuration + case "internal.Size", "config.Size": + return FieldTypeSize + } + return FieldTypeFieldConfig + } + return FieldTypeUnknown +} + +func getFieldTypeFromStructField(structField reflect.StructField) FieldType { + fieldName := structField.Name + ft := structField.Type + result := getFieldType(ft) + if result == FieldTypeUnknown { + panic(fmt.Sprintf("unknown type, name: %q, string: %q", fieldName, ft.String())) + } + return result +} + +func isInternalStructFieldType(t reflect.Type) bool { + switch t.String() { + case "internal.Duration", "config.Duration": + return true + case "internal.Size", "config.Size": + return true + default: + return false + } +} + +func idToString(id uint64) models.PluginID { + return models.PluginID(fmt.Sprintf("%016x", id)) +} + +// make sure these models implement RunningPlugin +var _ models.RunningPlugin = &models.RunningProcessor{} +var _ models.RunningPlugin = &models.RunningAggregator{} +var _ models.RunningPlugin = &models.RunningInput{} +var _ models.RunningPlugin = &models.RunningOutput{} + +type unwrappable interface { + Unwrap() telegraf.Processor +} diff --git a/plugins/configs/api/controller_test.go b/plugins/configs/api/controller_test.go new file mode 100644 index 0000000000000..022fab4208d40 --- /dev/null +++ b/plugins/configs/api/controller_test.go @@ -0,0 +1,707 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf/agent" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" + _ "github.com/influxdata/telegraf/plugins/aggregators/all" + "github.com/influxdata/telegraf/plugins/common/kafka" + "github.com/influxdata/telegraf/plugins/common/tls" + "github.com/influxdata/telegraf/plugins/inputs" + _ "github.com/influxdata/telegraf/plugins/inputs/all" + "github.com/influxdata/telegraf/plugins/inputs/kafka_consumer" + _ "github.com/influxdata/telegraf/plugins/outputs/all" + _ "github.com/influxdata/telegraf/plugins/processors/all" + // _ "github.com/influxdata/telegraf/plugins/inputs/cpu" + // _ "github.com/influxdata/telegraf/plugins/outputs/file" + // _ "github.com/influxdata/telegraf/plugins/processors/rename" +) + +// TestListPluginTypes tests that the config api can scrape all existing plugins +// for type information to build a schema. +func TestListPluginTypes(t *testing.T) { + cfg := config.NewConfig() // initalizes API + a := agent.NewAgent(context.Background(), cfg) + + api := newAPI(context.Background(), context.Background(), cfg, a) + + pluginConfigs := api.ListPluginTypes() + require.Greater(t, len(pluginConfigs), 10) + // b, _ := json.Marshal(pluginConfigs) + // fmt.Println(string(b)) + + // find the gnmi plugin + var gnmi PluginConfigTypeInfo + for _, conf := range pluginConfigs { + if conf.Name == "inputs.gnmi" { + gnmi = conf + break + } + } + + // find the cloudwatch plugin + var cloudwatch PluginConfigTypeInfo + for _, conf := range pluginConfigs { + if conf.Name == "inputs.cloudwatch" { + cloudwatch = conf + break + } + } + + // validate a slice of objects + require.EqualValues(t, "array", gnmi.Config["Subscriptions"].Type) + require.EqualValues(t, "object", gnmi.Config["Subscriptions"].SubType) + require.NotNil(t, gnmi.Config["Subscriptions"].SubFields) + require.EqualValues(t, "string", gnmi.Config["Subscriptions"].SubFields["Name"].Type) + + // validate a slice of pointer objects + require.EqualValues(t, "array", cloudwatch.Config["Metrics"].Type) + require.EqualValues(t, "object", cloudwatch.Config["Metrics"].SubType) + require.NotNil(t, cloudwatch.Config["Metrics"].SubFields) + require.EqualValues(t, "array", cloudwatch.Config["Metrics"].SubFields["StatisticExclude"].Type) + require.EqualValues(t, "array", cloudwatch.Config["Metrics"].SubFields["MetricNames"].Type) + + // validate a map of strings + require.EqualValues(t, "map", gnmi.Config["Aliases"].Type) + require.EqualValues(t, "string", gnmi.Config["Aliases"].SubType) + + // check a default value + require.EqualValues(t, "proto", gnmi.Config["Encoding"].Default) + require.EqualValues(t, 10*1e9, gnmi.Config["Redial"].Default) + + // check anonymous composed fields + require.EqualValues(t, "bool", gnmi.Config["InsecureSkipVerify"].Type) +} + +func TestInputPluginLifecycle(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + outputCtx, outputCancel := context.WithCancel(context.Background()) + defer outputCancel() + + cfg := config.NewConfig() // initalizes API + a := agent.NewAgent(ctx, cfg) + api := newAPI(ctx, outputCtx, cfg, a) + + go a.RunWithAPI(outputCancel) + + // create + newPluginID, err := api.CreatePlugin(PluginConfigCreate{ + Name: "inputs.cpu", + Config: map[string]interface{}{ + "percpu": true, + "totalcpu": true, + "collect_cpu_time": true, + "report_active": true, + }, + }, "") + require.NoError(t, err) + require.NotZero(t, len(newPluginID)) + + // get plugin status + waitForStatus(t, api, newPluginID, "running", 20*time.Second) + + // list running + runningPlugins := api.ListRunningPlugins() + require.Len(t, runningPlugins, 1) + + status := api.GetPluginStatus(newPluginID) + require.Equal(t, "running", status.String()) + // delete + err = api.DeletePlugin(newPluginID) + require.NoError(t, err) + + waitForStatus(t, api, newPluginID, "dead", 300*time.Millisecond) + + // get plugin status until dead + status = api.GetPluginStatus(newPluginID) + require.Equal(t, "dead", status.String()) + + // list running should have none + runningPlugins = api.ListRunningPlugins() + require.Len(t, runningPlugins, 0) + require.Equal(t, []Plugin{}, runningPlugins) +} + +func TestAllPluginLifecycle(t *testing.T) { + runCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + outputCtx, outputCancel := context.WithCancel(context.Background()) + defer outputCancel() + + cfg := config.NewConfig() + a := agent.NewAgent(context.Background(), cfg) + + api := newAPI(runCtx, outputCtx, cfg, a) + + go a.RunWithAPI(outputCancel) + + // create + pluginIDs := []models.PluginID{} + newPluginID, err := api.CreatePlugin(PluginConfigCreate{ + Name: "inputs.cpu", + Config: map[string]interface{}{}, + }, "") + pluginIDs = append(pluginIDs, newPluginID) + require.NoError(t, err) + require.NotZero(t, len(newPluginID)) + + newPluginID, err = api.CreatePlugin(PluginConfigCreate{ + Name: "processors.rename", + Config: map[string]interface{}{ + "replace": []map[string]interface{}{{ + "tag": "hostname", + "dest": "a_host", + }}, + }, + }, "") + require.NoError(t, err) + pluginIDs = append(pluginIDs, newPluginID) + require.NotZero(t, len(newPluginID)) + + newPluginID, err = api.CreatePlugin(PluginConfigCreate{ + Name: "outputs.file", + Config: map[string]interface{}{ + "files": []string{"stdout"}, + }, + }, "") + pluginIDs = append(pluginIDs, newPluginID) + require.NoError(t, err) + require.NotZero(t, len(newPluginID)) + + for _, id := range pluginIDs { + waitForStatus(t, api, id, "running", 10*time.Second) + } + + // list running + runningPlugins := api.ListRunningPlugins() + require.Len(t, runningPlugins, 3) + + time.Sleep(5 * time.Second) + + // delete + for _, id := range pluginIDs { + err = api.DeletePlugin(id) + require.NoError(t, err) + } + + for _, id := range pluginIDs { + waitForStatus(t, api, id, "dead", 300*time.Millisecond) + } + + // plugins might not be delisted immediately.. loop until done + for { + // list running should have none + runningPlugins = api.ListRunningPlugins() + if len(runningPlugins) == 0 { + break + } + time.Sleep(50 * time.Millisecond) + } +} + +func waitForStatus(t *testing.T, api *api, newPluginID models.PluginID, waitStatus string, timeout time.Duration) { + timeoutAt := time.Now().Add(timeout) + for timeoutAt.After(time.Now()) { + status := api.GetPluginStatus(newPluginID) + if status.String() == waitStatus { + return + } + time.Sleep(10 * time.Millisecond) + } + require.FailNow(t, "timed out waiting for status "+waitStatus) +} + +func TestSetFieldConfig(t *testing.T) { + creator := inputs.Inputs["kafka_consumer"] + cfg := map[string]interface{}{ + "name": "alias", + "alias": "bar", + "interval": "30s", + "collection_jitter": "5s", + "precision": "1ms", + "name_override": "my", + "measurement_prefix": "prefix_", + "measurement_suffix": "_suffix", + "tags": map[string]interface{}{ + "tag1": "value", + }, + "filter": map[string]interface{}{ + "namedrop": []string{"namedrop"}, + "namepass": []string{"namepass"}, + "fielddrop": []string{"fielddrop"}, + "fieldpass": []string{"fieldpass"}, + "tagdrop": []map[string]interface{}{{ + "name": "tagfilter", + "filter": []string{"filter"}, + }}, + "tagpass": []map[string]interface{}{{ + "name": "tagpass", + "filter": []string{"tagpassfilter"}, + }}, + "tagexclude": []string{"tagexclude"}, + "taginclude": []string{"taginclude"}, + }, + "brokers": []string{"localhost:9092"}, + "topics": []string{"foo"}, + "topic_tag": "foo", + "client_id": "tg123", + "tls_ca": "/etc/telegraf/ca.pem", + "tls_cert": "/etc/telegraf/cert.pem", + "tls_key": "/etc/telegraf/key.pem", + "insecure_skip_verify": true, + "sasl_mechanism": "SCRAM-SHA-256", + "sasl_version": 1, + "compression_codec": 1, + "sasl_username": "Some-Username", + "data_format": "influx", + } + i := creator() + err := setFieldConfig(cfg, i) + require.NoError(t, err) + expect := &kafka_consumer.KafkaConsumer{ + Brokers: []string{"localhost:9092"}, + Topics: []string{"foo"}, + TopicTag: "foo", + ReadConfig: kafka.ReadConfig{ + Config: kafka.Config{ + ClientID: "tg123", + CompressionCodec: 1, + SASLAuth: kafka.SASLAuth{ + SASLUsername: "Some-Username", + SASLMechanism: "SCRAM-SHA-256", + SASLVersion: intptr(1), + }, + ClientConfig: tls.ClientConfig{ + TLSCA: "/etc/telegraf/ca.pem", + TLSCert: "/etc/telegraf/cert.pem", + TLSKey: "/etc/telegraf/key.pem", + InsecureSkipVerify: true, + }, + }, + }, + } + + require.Equal(t, expect, i) + + icfg := &models.InputConfig{} + err = setFieldConfig(cfg, icfg) + require.NoError(t, err) + expected := &models.InputConfig{ + Name: "alias", + Alias: "bar", + Interval: 30 * time.Second, + CollectionJitter: 5 * time.Second, + Precision: 1 * time.Millisecond, + NameOverride: "my", + MeasurementPrefix: "prefix_", + MeasurementSuffix: "_suffix", + Tags: map[string]string{ + "tag1": "value", + }, + Filter: models.Filter{ + NameDrop: []string{"namedrop"}, + NamePass: []string{"namepass"}, + FieldDrop: []string{"fielddrop"}, + FieldPass: []string{"fieldpass"}, + TagDrop: []models.TagFilter{{ + Name: "tagfilter", + Filter: []string{"filter"}, + }}, + TagPass: []models.TagFilter{{ + Name: "tagpass", + Filter: []string{"tagpassfilter"}, + }}, + TagExclude: []string{"tagexclude"}, + TagInclude: []string{"taginclude"}, + }, + } + + require.Equal(t, expected, icfg) +} + +func TestExampleWorstPlugin(t *testing.T) { + input := map[string]interface{}{ + "elapsed": "3s", + "elapsed2": "4s", + "read_timeout": "5s", + "size1": "8MiB", + "size2": "9MiB", + "pointer_struct": map[string]interface{}{ + "field": "f", + }, + "b": true, + "i": 1, + "i8": 2, + "i32": 3, + "u8": 4, + "f": 5.0, + "pf": 6.0, + "ps": "I am a string pointer", + // type Header map[string][]string + "header": map[string]interface{}{ + "Content-Type": []interface{}{ + "json/application", "text/html", + }, + }, + "fields": map[string]interface{}{ + "field1": "field1", + "field2": 1, + "field3": float64(5), + }, + "reserved_keys": map[string]bool{ + "key": true, + }, + "string_to_number": map[string][]map[string]float64{ + "s": { + { + "n": 1.0, + }, + }, + }, + "clean": []map[string]interface{}{ + { + "field": "fieldtest", + }, + }, + "templates": []map[string]interface{}{ + { + "tag": "tagtest", + }, + }, + "value": "string", + "device_tags": map[string][]map[string]string{ + "s": { + { + "n": "1.0", + }, + }, + }, + "percentiles": []interface{}{ + 1, + }, + "float_percentiles": []interface{}{ + 1.0, + }, + "map_of_structs": map[string]interface{}{ + "src": map[string]interface{}{ + "dest": "d", + }, + }, + "command": []interface{}{ + "string", + 1, + 2.0, + }, + "tag_slice": []interface{}{ + []interface{}{ + "s", + }, + }, + "address": []interface{}{ + 1, + }, + } + readTimeout := config.Duration(5 * time.Second) + b := true + i := 1 + f := float64(6) + s := "I am a string pointer" + header := http.Header{ + "Content-Type": []string{"json/application", "text/html"}, + } + expected := ExampleWorstPlugin{ + Elapsed: config.Duration(3 * time.Second), + Elapsed2: config.Duration(4 * time.Second), + ReadTimeout: &readTimeout, + Size1: config.Size(8 * 1024 * 1024), + Size2: config.Size(9 * 1024 * 1024), + PointerStruct: &baseopts{Field: "f"}, + B: &b, + I: &i, + I8: 2, + I32: 3, + U8: 4, + F: 5, + PF: &f, + PS: &s, + Header: header, + DefaultFieldsSets: map[string]interface{}{ + "field1": "field1", + "field2": 1, + "field3": float64(5), + }, + ReservedKeys: map[string]bool{ + "key": true, + }, + StringToNumber: map[string][]map[string]float64{ + "s": { + { + "n": 1.0, + }, + }, + }, + Clean: []baseopts{ + {Field: "fieldtest"}, + }, + Templates: []*baseopts{ + {Tag: "tagtest"}, + }, + Value: "string", + DeviceTags: map[string][]map[string]string{ + "s": { + { + "n": "1.0", + }, + }, + }, + Percentiles: []int64{ + 1, + }, + FloatPercentiles: []float64{1.0}, + MapOfStructs: map[string]baseopts{ + "src": { + Dest: "d", + }, + }, + Command: []interface{}{ + "string", + 1, + 2.0, + }, + TagSlice: [][]string{ + {"s"}, + }, + Address: []uint16{ + 1, + }, + } + + actual := ExampleWorstPlugin{} + err := setFieldConfig(input, &actual) + require.NoError(t, err) + require.Equal(t, expected, actual) +} + +type ExampleWorstPlugin struct { + Elapsed config.Duration + Elapsed2 config.Duration + ReadTimeout *config.Duration + Size1 config.Size + Size2 config.Size + PointerStruct *baseopts + B *bool + I *int + I8 int8 + I32 int32 + U8 uint8 + F float64 + PF *float64 + PS *string + Header http.Header + DefaultFieldsSets map[string]interface{} `toml:"fields"` + ReservedKeys map[string]bool + StringToNumber map[string][]map[string]float64 + Clean []baseopts + Templates []*baseopts + Value interface{} `json:"value"` + DeviceTags map[string][]map[string]string + Percentiles []int64 + FloatPercentiles []float64 + MapOfStructs map[string]baseopts + Command []interface{} + TagSlice [][]string + Address []uint16 `toml:"address"` +} + +type baseopts struct { + Field string + Tag string + Dest string +} + +func intptr(i int) *int { + return &i +} + +func TestGetFieldConfigValues(t *testing.T) { + readTimeout := config.Duration(5 * time.Second) + b := true + i := 1 + f := float64(6) + s := "I am a string pointer" + header := http.Header{ + "Content-Type": []string{"json/application", "text/html"}, + } + input := ExampleWorstPlugin{ + Elapsed: config.Duration(3 * time.Second), + Elapsed2: config.Duration(4 * time.Second), + ReadTimeout: &readTimeout, + Size1: config.Size(8 * 1024 * 1024), + Size2: config.Size(9 * 1024 * 1024), + PointerStruct: &baseopts{Field: "f"}, + B: &b, + I: &i, + I8: 2, + I32: 3, + U8: 4, + F: 5, + PF: &f, + PS: &s, + Header: header, + DefaultFieldsSets: map[string]interface{}{ + "field1": "field1", + "field2": 1, + "field3": float64(5), + }, + ReservedKeys: map[string]bool{ + "key": true, + }, + StringToNumber: map[string][]map[string]float64{ + "s": { + { + "n": 1.0, + }, + }, + }, + Clean: []baseopts{ + {Field: "fieldtest"}, + }, + Templates: []*baseopts{ + {Tag: "tagtest"}, + }, + Value: "string", + DeviceTags: map[string][]map[string]string{ + "s": { + { + "n": "1.0", + }, + }, + }, + Percentiles: []int64{ + 1, + }, + FloatPercentiles: []float64{1.0}, + MapOfStructs: map[string]baseopts{ + "src": { + Dest: "d", + }, + }, + Command: []interface{}{ + "string", + 1, + 2.0, + }, + TagSlice: [][]string{ + {"s"}, + }, + Address: []uint16{ + 1, + }, + } + expected := map[string]interface{}{ + "elapsed": "3s", + "elapsed2": "4s", + "read_timeout": "5s", + "size1": "8MiB", + "size2": "9MiB", + "pointer_struct": map[string]interface{}{ + "field": "f", + "dest": "", + "tag": "", + }, + "b": true, + "i": 1, + "i8": 2, + "i32": 3, + "u8": 4, + "f": 5.0, + "pf": 6.0, + "ps": "I am a string pointer", + // type Header map[string][]string + "header": map[string]interface{}{ + "Content-Type": []interface{}{ + "json/application", "text/html", + }, + }, + "fields": map[string]interface{}{ + "field1": "field1", + "field2": 1, + "field3": float64(5), + }, + "reserved_keys": map[string]bool{ + "key": true, + }, + "string_to_number": map[string][]map[string]float64{ + "s": { + { + "n": 1.0, + }, + }, + }, + "clean": []map[string]interface{}{ + { + "field": "fieldtest", + "dest": "", + "tag": "", + }, + }, + "templates": []map[string]interface{}{ + { + "tag": "tagtest", + "dest": "", + "field": "", + }, + }, + "value": "string", + "device_tags": map[string][]map[string]string{ + "s": { + { + "n": "1.0", + }, + }, + }, + "percentiles": []interface{}{ + 1, + }, + "float_percentiles": []interface{}{ + 1.0, + }, + "map_of_structs": map[string]interface{}{ + "src": map[string]interface{}{ + "dest": "d", + "field": "", + "tag": "", + }, + }, + "command": []interface{}{ + "string", + 1, + 2.0, + }, + "tag_slice": []interface{}{ + []interface{}{ + "s", + }, + }, + "address": []interface{}{ + 1, + }, + } + actual := map[string]interface{}{} + getFieldConfigValuesFromStruct(input, actual) + // require.Equal(t, expected, actual) + expJSON, _ := json.Marshal(expected) + actualJSON, _ := json.Marshal(actual) + require.Equal(t, string(expJSON), string(actualJSON)) +} diff --git a/plugins/configs/api/listener.go b/plugins/configs/api/listener.go new file mode 100644 index 0000000000000..bff7b3d9bb888 --- /dev/null +++ b/plugins/configs/api/listener.go @@ -0,0 +1,190 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" // nolint:revive + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/models" +) + +var ( + ErrClient = errors.New("error") + ErrBadRequest = fmt.Errorf("%w bad request", ErrClient) + ErrNotFound = fmt.Errorf("%w not found", ErrClient) +) + +type ConfigAPIService struct { + server *http.Server + api *api + Log telegraf.Logger +} + +func newConfigAPIService(server *http.Server, api *api, logger telegraf.Logger) *ConfigAPIService { + service := &ConfigAPIService{ + server: server, + api: api, + Log: logger, + } + server.Handler = service.mux() + return service +} + +// nolint:revive +func (s *ConfigAPIService) mux() *mux.Router { + m := mux.NewRouter() + m.HandleFunc("/status", s.status).Methods("GET") + m.HandleFunc("/plugins/create", s.createPlugin).Methods("POST") + m.HandleFunc("/plugins/{id:[0-9a-f]+}/status", s.pluginStatus).Methods("GET") + m.HandleFunc("/plugins/list", s.listPlugins).Methods("GET") + m.HandleFunc("/plugins/running", s.runningPlugins).Methods("GET") + m.HandleFunc("/plugins/{id:[0-9a-f]+}", s.deleteOrUpdatePlugin).Methods("DELETE", "PUT") + return m +} + +func (s *ConfigAPIService) status(w http.ResponseWriter, req *http.Request) { + if req.Body != nil { + defer req.Body.Close() + } + _, err := w.Write([]byte("ok")) + if err != nil { + log.Printf("W! error writing to connection: %v", err) + return + } +} + +func (s *ConfigAPIService) createPlugin(w http.ResponseWriter, req *http.Request) { + if req.Body != nil { + defer req.Body.Close() + } + cfg := PluginConfigCreate{} + + dec := json.NewDecoder(req.Body) + if err := dec.Decode(&cfg); err != nil { + s.renderError(fmt.Errorf("%w: decode failed %v", ErrBadRequest, err), w) + return + } + id, err := s.api.CreatePlugin(cfg, "") + if err != nil { + s.renderError(err, w) + return + } + w.Header().Set("Content-Type", "application/json") + _, err = w.Write([]byte(fmt.Sprintf(`{"id": "%s"}`, id))) + if err != nil { + log.Printf("W! error writing to connection: %v", err) + return + } +} + +func (s *ConfigAPIService) renderError(err error, w http.ResponseWriter) { + if errors.Is(err, ErrBadRequest) { + s.Log.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } else if errors.Is(err, ErrNotFound) { + s.Log.Error(err) + w.WriteHeader(http.StatusNotFound) + return + } + s.Log.Error(err) + w.WriteHeader(http.StatusInternalServerError) +} + +func (s *ConfigAPIService) Start() { + go func() { + _ = s.server.ListenAndServe() + }() +} + +func (s *ConfigAPIService) listPlugins(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + typeInfo := s.api.ListPluginTypes() + + bytes, err := json.Marshal(typeInfo) + if err != nil { + s.renderError(fmt.Errorf("marshal failed %w", err), w) + return + } + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(bytes) + if err != nil { + log.Printf("W! error writing to connection: %v", err) + return + } +} + +func (s *ConfigAPIService) runningPlugins(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + plugins := s.api.ListRunningPlugins() + + bytes, err := json.Marshal(plugins) + if err != nil { + s.renderError(fmt.Errorf("marshal failed %w", err), w) + return + } + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(bytes) + if err != nil { + log.Printf("W! error writing to connection: %v", err) + return + } +} + +func (s *ConfigAPIService) pluginStatus(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + id := mux.Vars(req)["id"] + if len(id) == 0 { + w.WriteHeader(http.StatusBadRequest) + return + } + state := s.api.GetPluginStatus(models.PluginID(id)) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(fmt.Sprintf(`{"status": %q}`, state.String()))) + if err != nil { + log.Printf("W! error writing to connection: %v", err) + return + } +} + +func (s *ConfigAPIService) Stop() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := s.server.Shutdown(ctx); err != nil { + log.Printf("W! [configapi] error on shutdown: %s", err) + } +} + +func (s *ConfigAPIService) deleteOrUpdatePlugin(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case "DELETE": + s.deletePlugin(w, req) + case "PUT": + s.updatePlugin(w, req) + default: + w.WriteHeader(http.StatusBadRequest) + } +} + +func (s *ConfigAPIService) deletePlugin(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + id := mux.Vars(req)["id"] + if len(id) == 0 { + w.WriteHeader(http.StatusBadRequest) + return + } + if err := s.api.DeletePlugin(models.PluginID(id)); err != nil { + s.renderError(fmt.Errorf("delete plugin %w", err), w) + } + w.WriteHeader(http.StatusOK) +} + +func (s *ConfigAPIService) updatePlugin(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} diff --git a/plugins/configs/api/listener_test.go b/plugins/configs/api/listener_test.go new file mode 100644 index 0000000000000..430bdf55610c7 --- /dev/null +++ b/plugins/configs/api/listener_test.go @@ -0,0 +1,246 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/influxdata/telegraf/agent" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestStatus(t *testing.T) { + s := &ConfigAPIService{} + srv := httptest.NewServer(s.mux()) + + resp, err := http.Get(srv.URL + "/status") + require.NoError(t, err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.EqualValues(t, "ok", body) + require.EqualValues(t, 200, resp.StatusCode) +} + +func TestStartPlugin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + outputCtx, outputCancel := context.WithCancel(context.Background()) + defer outputCancel() + + c := config.NewConfig() + a := agent.NewAgent(ctx, c) + api := newAPI(ctx, outputCtx, c, a) + go a.RunWithAPI(outputCancel) + + s := &ConfigAPIService{ + api: api, + } + srv := httptest.NewServer(s.mux()) + + buf := bytes.NewBufferString(`{ + "name": "inputs.file", + "config": { + "files": ["testdata.lp"], + "data_format": "influx" + } +}`) + resp, err := http.Post(srv.URL+"/plugins/create", "application/json", buf) + require.NoError(t, err) + createResp := struct { + ID string + }{} + require.EqualValues(t, 200, resp.StatusCode) + err = json.NewDecoder(resp.Body).Decode(&createResp) + require.NoError(t, err) + _ = resp.Body.Close() + + require.Regexp(t, `^[\da-f]{8}\d{8}$`, createResp.ID) + + statusResp := struct { + Status string + Reason string + }{} + + for statusResp.Status != "running" { + resp, err = http.Get(srv.URL + "/plugins/" + createResp.ID + "/status") + require.NoError(t, err) + + require.EqualValues(t, 200, resp.StatusCode) + err = json.NewDecoder(resp.Body).Decode(&statusResp) + require.NoError(t, err) + _ = resp.Body.Close() + } + + require.EqualValues(t, "running", statusResp.Status) + + resp, err = http.Get(srv.URL + "/plugins/list") + require.NoError(t, err) + require.EqualValues(t, 200, resp.StatusCode) + listResp := []PluginConfigTypeInfo{} + err = json.NewDecoder(resp.Body).Decode(&listResp) + require.NoError(t, err) + _ = resp.Body.Close() + + if len(listResp) < 20 { + require.FailNow(t, "expected there to be more than 20 plugins loaded, was only", len(listResp)) + } + + resp, err = http.Get(srv.URL + "/plugins/running") + require.NoError(t, err) + require.EqualValues(t, 200, resp.StatusCode) + runningList := []Plugin{} + err = json.NewDecoder(resp.Body).Decode(&runningList) + require.NoError(t, err) + _ = resp.Body.Close() + + if len(runningList) != 1 { + require.FailNow(t, "expected there to be 1 running plugin, was", len(runningList)) + } +} + +func TestStopPlugin(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + outputCtx, outputCancel := context.WithCancel(context.Background()) + defer outputCancel() + + c := config.NewConfig() + a := agent.NewAgent(ctx, c) + api := newAPI(ctx, outputCtx, c, a) + go a.RunWithAPI(outputCancel) + + s := &ConfigAPIService{ + api: api, + Log: testutil.Logger{}, + } + srv := httptest.NewServer(s.mux()) + + // start plugin + buf := bytes.NewBufferString(`{ + "name": "inputs.cpu", + "config": { + "percpu": true + } +}`) + resp, err := http.Post(srv.URL+"/plugins/create", "application/json", buf) + require.NoError(t, err) + createResp := struct { + ID string + }{} + require.EqualValues(t, 200, resp.StatusCode) + err = json.NewDecoder(resp.Body).Decode(&createResp) + require.NoError(t, err) + _ = resp.Body.Close() + + require.Regexp(t, `^[\da-f]{8}\d{8}$`, createResp.ID) + + waitForStatus(t, api, models.PluginID(createResp.ID), models.PluginStateRunning.String(), 2*time.Second) + + resp, err = http.Get(srv.URL + "/plugins/running") + require.NoError(t, err) + require.EqualValues(t, 200, resp.StatusCode) + runningList := []Plugin{} + err = json.NewDecoder(resp.Body).Decode(&runningList) + require.NoError(t, err) + _ = resp.Body.Close() + + // confirm plugin is running + if len(runningList) != 1 { + require.FailNow(t, "expected there to be 1 running plugin, was", len(runningList)) + } + + // stop plugin + client := &http.Client{} + req, err := http.NewRequest("DELETE", srv.URL+"/plugins/"+createResp.ID, nil) + require.NoError(t, err) + + resp, err = client.Do(req) + require.NoError(t, err) + _ = resp.Body.Close() + + require.EqualValues(t, 200, resp.StatusCode) + require.NoError(t, err) + + waitForStatus(t, api, models.PluginID(createResp.ID), models.PluginStateDead.String(), 2*time.Second) + + resp, err = http.Get(srv.URL + "/plugins/running") + require.NoError(t, err) + require.EqualValues(t, 200, resp.StatusCode) + runningList = []Plugin{} + err = json.NewDecoder(resp.Body).Decode(&runningList) + require.NoError(t, err) + _ = resp.Body.Close() + + // confirm plugin has stopped + if len(runningList) >= 1 { + require.FailNow(t, "expected there to be no running plugin, was", len(runningList)) + } + + // try to delete a plugin which was already been deleted + req, err = http.NewRequest("DELETE", srv.URL+"/plugins/"+createResp.ID, nil) + require.NoError(t, err) + + resp, err = client.Do(req) + require.NoError(t, err) + _ = resp.Body.Close() + + require.EqualValues(t, 404, resp.StatusCode) + require.NoError(t, err) +} + +func TestStatusCodes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + outputCtx, outputCancel := context.WithCancel(context.Background()) + defer outputCancel() + + c := config.NewConfig() + a := agent.NewAgent(ctx, c) + api := newAPI(ctx, outputCtx, c, a) + go a.RunWithAPI(outputCancel) + + s := &ConfigAPIService{ + api: api, + Log: testutil.Logger{}, + } + srv := httptest.NewServer(s.mux()) + + // Error finding plugin with wrong name + buf := bytes.NewBufferString(`{ + "name": "inputs.blah", + "config": { + "files": ["testdata.lp"], + "data_format": "influx" + } + }`) + resp, err := http.Post(srv.URL+"/plugins/create", "application/json", buf) + require.NoError(t, err) + _ = resp.Body.Close() + require.EqualValues(t, 404, resp.StatusCode) + require.NoError(t, err) + + // Error creating plugin with wrong data format + buf = bytes.NewBufferString(`{ + "name": "inputs.file", + "config": { + "files": ["testdata.lp"], + "data_format": "blah" + } + }`) + resp, err = http.Post(srv.URL+"/plugins/create", "application/json", buf) + require.NoError(t, err) + _ = resp.Body.Close() + require.EqualValues(t, 400, resp.StatusCode) + require.NoError(t, err) +} diff --git a/plugins/configs/api/plugin.go b/plugins/configs/api/plugin.go new file mode 100644 index 0000000000000..e8c7cf2f018c5 --- /dev/null +++ b/plugins/configs/api/plugin.go @@ -0,0 +1,121 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" + "github.com/influxdata/telegraf/plugins/common/tls" + "github.com/influxdata/telegraf/plugins/configs" +) + +type ConfigAPIPlugin struct { + ServiceAddress string `toml:"service_address"` + Storage config.StoragePlugin `toml:"storage"` + tls.ServerConfig + + api *api + server *ConfigAPIService + + Log telegraf.Logger `toml:"-"` + plugins []PluginConfig + pluginsMutex sync.Mutex +} + +func (a *ConfigAPIPlugin) GetName() string { + return "api" +} + +// Init initializes the config api plugin. +// nolint:revive +func (a *ConfigAPIPlugin) Init(ctx context.Context, outputCtx context.Context, cfg *config.Config, agent config.AgentController) error { + a.api = newAPI(ctx, outputCtx, cfg, agent) + if a.Storage == nil { + a.Log.Warn("initializing config-api without storage, changes via the api will not be persisted.") + } else { + if err := a.Storage.Init(); err != nil { + return fmt.Errorf("initializing storage: %w", err) + } + + if err := a.Storage.Load("config-api", "plugins", &a.plugins); err != nil { + return fmt.Errorf("loading plugin state: %w", err) + } + } + + a.Log.Info(fmt.Sprintf("Loading %d stored plugins", len(a.plugins))) + for _, plug := range a.plugins { + id, err := a.api.CreatePlugin(plug.PluginConfigCreate, models.PluginID(plug.ID)) + if err != nil { + a.Log.Errorf("Couldn't recreate plugin %q: %s", id, err) + } + } + + a.api.OnPluginAdded(func(p PluginConfig) { + a.pluginsMutex.Lock() + defer a.pluginsMutex.Unlock() + a.plugins = append(a.plugins, p) + if err := a.Storage.Save("config-api", "plugins", &a.plugins); err != nil { + a.Log.Error("saving plugin state: " + err.Error()) + } + }) + a.api.OnPluginRemoved(func(p PluginConfig) { + a.pluginsMutex.Lock() + defer a.pluginsMutex.Unlock() + changed := false + for i, plug := range a.plugins { + if plug.ID == p.ID { + a.plugins = append(a.plugins[:i], a.plugins[i+1:]...) + changed = true + } + } + if changed { + if err := a.Storage.Save("config-api", "plugins", &a.plugins); err != nil { + a.Log.Error("saving plugin state: " + err.Error()) + } + } + }) + + // start listening for HTTP requests + tlsConfig, err := a.TLSConfig() + if err != nil { + return err + } + if a.ServiceAddress == "" { + a.ServiceAddress = ":7551" + } + a.server = newConfigAPIService(&http.Server{ + Addr: a.ServiceAddress, + TLSConfig: tlsConfig, + }, a.api, a.Log) + + a.server.Start() + return nil +} + +func (a *ConfigAPIPlugin) Close() error { + // shut down server + // stop accepting new requests + // wait until all requests finish + a.server.Stop() + + a.pluginsMutex.Lock() + defer a.pluginsMutex.Unlock() + + // store state + if a.Storage != nil { + if err := a.Storage.Save("config-api", "plugins", &a.plugins); err != nil { + return fmt.Errorf("saving plugin state: %w", err) + } + } + return nil +} + +func init() { + configs.Add("api", func() config.ConfigPlugin { + return &ConfigAPIPlugin{} + }) +} diff --git a/plugins/configs/api/testdata.lp b/plugins/configs/api/testdata.lp new file mode 100644 index 0000000000000..e056407e7305d --- /dev/null +++ b/plugins/configs/api/testdata.lp @@ -0,0 +1,5 @@ +sqlserver_requests,database_name=AdventureWorks2016,sql_instance=QDLP03:SQL2017,query_hash=aaa Somefield=10 +sqlserver_requests,database_name=AdventureWorks2016,sql_instance=QDLP03:SQL2017,query_hash=bbb Somefield=2 +sqlserver_requests,database_name=Northwind,sql_instance=QDLP03:SQL2017,query_hash=ccc Somefield=4 +sqlserver_requests,database_name=Northwind,sql_instance=QDLP03:SQL2017,query_hash=ddd Somefield=6 +sqlserver_cpu,sql_instance=QDLP03:SQL2017 other_process_cpu=8i,sqlserver_process_cpu=12i,system_idle_cpu=80i diff --git a/plugins/configs/registry.go b/plugins/configs/registry.go new file mode 100644 index 0000000000000..067f3743b9f6b --- /dev/null +++ b/plugins/configs/registry.go @@ -0,0 +1,7 @@ +package configs + +import "github.com/influxdata/telegraf/config" + +func Add(name string, creator config.ConfigCreator) { + config.ConfigPlugins[name] = creator +} diff --git a/plugins/inputs/execd/execd_test.go b/plugins/inputs/execd/execd_test.go index a8c8364394480..433f7d8413033 100644 --- a/plugins/inputs/execd/execd_test.go +++ b/plugins/inputs/execd/execd_test.go @@ -2,6 +2,7 @@ package execd import ( "bufio" + "context" "flag" "fmt" "os" @@ -13,6 +14,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/agent" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/models" @@ -29,10 +31,11 @@ func TestSettingConfigWorks(t *testing.T) { signal = "SIGHUP" ` conf := config.NewConfig() - require.NoError(t, conf.LoadConfigData([]byte(cfg))) + conf.SetAgent(&agenthelper.TestAgentController{}) + require.NoError(t, conf.LoadConfigData(context.Background(), context.Background(), []byte(cfg))) - require.Len(t, conf.Inputs, 1) - inp, ok := conf.Inputs[0].Input.(*Execd) + require.Len(t, conf.Inputs(), 1) + inp, ok := conf.Inputs()[0].Input.(*Execd) require.True(t, ok) require.EqualValues(t, []string{"a", "b", "c"}, inp.Command) require.EqualValues(t, 1*time.Minute, inp.RestartDelay) diff --git a/plugins/inputs/mongodb/mongodb.go b/plugins/inputs/mongodb/mongodb.go index 0366636200064..196249b03418f 100644 --- a/plugins/inputs/mongodb/mongodb.go +++ b/plugins/inputs/mongodb/mongodb.go @@ -135,6 +135,7 @@ func (m *MongoDB) Init() error { opts.ReadPreference = readpref.Nearest() } + // TODO: should not connect in Init() function! client, err := mongo.Connect(ctx, opts) if err != nil { return fmt.Errorf("unable to connect to MongoDB: %q", err) diff --git a/plugins/inputs/opcua/opcua_client_test.go b/plugins/inputs/opcua/opcua_client_test.go index ffa8521dd05a8..4217ec2b2ee63 100644 --- a/plugins/inputs/opcua/opcua_client_test.go +++ b/plugins/inputs/opcua/opcua_client_test.go @@ -1,11 +1,14 @@ package opcua_client import ( + "context" "fmt" "reflect" + "strings" "testing" "time" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -75,6 +78,10 @@ func MapOPCTag(tags OPCTags) (out NodeSettings) { } func TestConfig(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode. You can remove this skip after Init() doesn't try to connect") + } + toml := ` [[inputs.opcua]] name = "localhost" @@ -107,12 +114,16 @@ nodes = [{name="name4", identifier="4000", tags=[["tag1", "override"]]}] ` c := config.NewConfig() - err := c.LoadConfigData([]byte(toml)) - require.NoError(t, err) + c.SetAgent(&agenthelper.TestAgentController{}) + err := c.LoadConfigData(context.Background(), context.Background(), []byte(toml)) + // opcua shouldn't be trying to connect in init. + if !strings.Contains(err.Error(), "connection refused") { + require.NoError(t, err) + } - require.Len(t, c.Inputs, 1) + require.Len(t, c.Inputs(), 1) - o, ok := c.Inputs[0].Input.(*OpcUA) + o, ok := c.Inputs()[0].Input.(*OpcUA) require.True(t, ok) require.Len(t, o.RootNodes, 2) diff --git a/plugins/inputs/phpfpm/phpfpm_test.go b/plugins/inputs/phpfpm/phpfpm_test.go index c3a3f29f570f5..50d8d604efb5b 100644 --- a/plugins/inputs/phpfpm/phpfpm_test.go +++ b/plugins/inputs/phpfpm/phpfpm_test.go @@ -274,7 +274,7 @@ func TestPhpFpmGeneratesMetrics_From_Socket_Custom_Status_Path(t *testing.T) { //When not passing server config, we default to localhost //We just want to make sure we did request stat from localhost func TestPhpFpmDefaultGetFromLocalhost(t *testing.T) { - r := &phpfpm{} + r := &phpfpm{Urls: []string{"http://bad.localhost:62001/status"}} require.NoError(t, r.Init()) @@ -282,7 +282,7 @@ func TestPhpFpmDefaultGetFromLocalhost(t *testing.T) { err := acc.GatherError(r.Gather) require.Error(t, err) - assert.Contains(t, err.Error(), "127.0.0.1/status") + assert.Contains(t, err.Error(), "/status") } func TestPhpFpmGeneratesMetrics_Throw_Error_When_Fpm_Status_Is_Not_Responding(t *testing.T) { diff --git a/plugins/outputs/sumologic/sumologic_test.go b/plugins/outputs/sumologic/sumologic_test.go index 5ce502bab2c0e..b5564a74e2ff6 100644 --- a/plugins/outputs/sumologic/sumologic_test.go +++ b/plugins/outputs/sumologic/sumologic_test.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "fmt" "io" "io/ioutil" @@ -18,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/metric" @@ -432,11 +434,12 @@ func TestTOMLConfig(t *testing.T) { for _, tt := range testcases { t.Run(tt.name, func(t *testing.T) { c := config.NewConfig() + c.SetAgent(&agenthelper.TestAgentController{}) if tt.expectedError { - require.Error(t, c.LoadConfigData(tt.configBytes)) + require.Error(t, c.LoadConfigData(context.Background(), context.Background(), tt.configBytes)) } else { - require.NoError(t, c.LoadConfigData(tt.configBytes)) + require.NoError(t, c.LoadConfigData(context.Background(), context.Background(), tt.configBytes)) } }) } diff --git a/plugins/parsers/json_v2/parser_test.go b/plugins/parsers/json_v2/parser_test.go index f0f018034dc5b..213be5f3f6ab5 100644 --- a/plugins/parsers/json_v2/parser_test.go +++ b/plugins/parsers/json_v2/parser_test.go @@ -2,12 +2,14 @@ package json_v2_test import ( "bufio" + "context" "fmt" "io/ioutil" "os" "testing" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs/file" @@ -96,12 +98,13 @@ func TestData(t *testing.T) { return &file.File{} }) cfg := config.NewConfig() - err = cfg.LoadConfigData(buf) + cfg.SetAgent(&agenthelper.TestAgentController{}) + err = cfg.LoadConfigData(context.Background(), context.Background(), buf) require.NoError(t, err) // Gather the metrics from the input file configure acc := testutil.Accumulator{} - for _, i := range cfg.Inputs { + for _, i := range cfg.Inputs() { err = i.Init() require.NoError(t, err) err = i.Gather(&acc) diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index cc2102c9532d2..6c35a3971972f 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -66,107 +66,107 @@ type Parser interface { // and can be used to instantiate _any_ of the parsers. type Config struct { // Dataformat can be one of: json, influx, graphite, value, nagios - DataFormat string `toml:"data_format"` + DataFormat string `toml:"data_format" json:"data_format"` // Separator only applied to Graphite data. - Separator string `toml:"separator"` + Separator string `toml:"separator" json:"separator"` // Templates only apply to Graphite data. - Templates []string `toml:"templates"` + Templates []string `toml:"templates" json:"templates"` // TagKeys only apply to JSON data - TagKeys []string `toml:"tag_keys"` + TagKeys []string `toml:"tag_keys" json:"tag_keys"` // Array of glob pattern strings keys that should be added as string fields. - JSONStringFields []string `toml:"json_string_fields"` + JSONStringFields []string `toml:"json_string_fields" json:"json_string_fields"` - JSONNameKey string `toml:"json_name_key"` + JSONNameKey string `toml:"json_name_key" json:"json_name_key"` // MetricName applies to JSON & value. This will be the name of the measurement. - MetricName string `toml:"metric_name"` + MetricName string `toml:"metric_name" json:"metric_name"` // holds a gjson path for json parser - JSONQuery string `toml:"json_query"` + JSONQuery string `toml:"json_query" json:"json_query"` // key of time - JSONTimeKey string `toml:"json_time_key"` + JSONTimeKey string `toml:"json_time_key" json:"json_time_key"` // time format - JSONTimeFormat string `toml:"json_time_format"` + JSONTimeFormat string `toml:"json_time_format" json:"json_time_format"` // default timezone - JSONTimezone string `toml:"json_timezone"` + JSONTimezone string `toml:"json_timezone" json:"json_timezone"` // Whether to continue if a JSON object can't be coerced - JSONStrict bool `toml:"json_strict"` + JSONStrict bool `toml:"json_strict" json:"json_strict"` // Authentication file for collectd - CollectdAuthFile string `toml:"collectd_auth_file"` + CollectdAuthFile string `toml:"collectd_auth_file" json:"collectd_auth_file"` // One of none (default), sign, or encrypt - CollectdSecurityLevel string `toml:"collectd_security_level"` + CollectdSecurityLevel string `toml:"collectd_security_level" json:"collectd_security_level"` // Dataset specification for collectd - CollectdTypesDB []string `toml:"collectd_types_db"` + CollectdTypesDB []string `toml:"collectd_types_db" json:"collectd_types_db"` // whether to split or join multivalue metrics - CollectdSplit string `toml:"collectd_split"` + CollectdSplit string `toml:"collectd_split" json:"collectd_split"` // DataType only applies to value, this will be the type to parse value to - DataType string `toml:"data_type"` + DataType string `toml:"data_type" json:"data_type"` // DefaultTags are the default tags that will be added to all parsed metrics. - DefaultTags map[string]string `toml:"default_tags"` + DefaultTags map[string]string `toml:"default_tags" json:"default_tags"` // an optional json path containing the metric registry object // if left empty, the whole json object is parsed as a metric registry - DropwizardMetricRegistryPath string `toml:"dropwizard_metric_registry_path"` + DropwizardMetricRegistryPath string `toml:"dropwizard_metric_registry_path" json:"dropwizard_metric_registry_path"` // an optional json path containing the default time of the metrics // if left empty, the processing time is used - DropwizardTimePath string `toml:"dropwizard_time_path"` + DropwizardTimePath string `toml:"dropwizard_time_path" json:"dropwizard_time_path"` // time format to use for parsing the time field // defaults to time.RFC3339 - DropwizardTimeFormat string `toml:"dropwizard_time_format"` + DropwizardTimeFormat string `toml:"dropwizard_time_format" json:"dropwizard_time_format"` // an optional json path pointing to a json object with tag key/value pairs // takes precedence over DropwizardTagPathsMap - DropwizardTagsPath string `toml:"dropwizard_tags_path"` + DropwizardTagsPath string `toml:"dropwizard_tags_path" json:"dropwizard_tags_path"` // an optional map containing tag names as keys and json paths to retrieve the tag values from as values // used if TagsPath is empty or doesn't return any tags - DropwizardTagPathsMap map[string]string `toml:"dropwizard_tag_paths_map"` + DropwizardTagPathsMap map[string]string `toml:"dropwizard_tag_paths_map" json:"dropwizard_tag_paths_map"` //grok patterns - GrokPatterns []string `toml:"grok_patterns"` - GrokNamedPatterns []string `toml:"grok_named_patterns"` - GrokCustomPatterns string `toml:"grok_custom_patterns"` - GrokCustomPatternFiles []string `toml:"grok_custom_pattern_files"` - GrokTimezone string `toml:"grok_timezone"` - GrokUniqueTimestamp string `toml:"grok_unique_timestamp"` + GrokPatterns []string `toml:"grok_patterns" json:"grok_patterns"` + GrokNamedPatterns []string `toml:"grok_named_patterns" json:"grok_named_patterns"` + GrokCustomPatterns string `toml:"grok_custom_patterns" json:"grok_custom_patterns"` + GrokCustomPatternFiles []string `toml:"grok_custom_pattern_files" json:"grok_custom_pattern_files"` + GrokTimezone string `toml:"grok_timezone" json:"grok_timezone"` + GrokUniqueTimestamp string `toml:"grok_unique_timestamp" json:"grok_unique_timestamp"` //csv configuration - CSVColumnNames []string `toml:"csv_column_names"` - CSVColumnTypes []string `toml:"csv_column_types"` - CSVComment string `toml:"csv_comment"` - CSVDelimiter string `toml:"csv_delimiter"` - CSVHeaderRowCount int `toml:"csv_header_row_count"` - CSVMeasurementColumn string `toml:"csv_measurement_column"` - CSVSkipColumns int `toml:"csv_skip_columns"` - CSVSkipRows int `toml:"csv_skip_rows"` - CSVTagColumns []string `toml:"csv_tag_columns"` - CSVTimestampColumn string `toml:"csv_timestamp_column"` - CSVTimestampFormat string `toml:"csv_timestamp_format"` - CSVTimezone string `toml:"csv_timezone"` - CSVTrimSpace bool `toml:"csv_trim_space"` - CSVSkipValues []string `toml:"csv_skip_values"` + CSVColumnNames []string `toml:"csv_column_names" json:"csv_column_names"` + CSVColumnTypes []string `toml:"csv_column_types" json:"csv_column_types"` + CSVComment string `toml:"csv_comment" json:"csv_comment"` + CSVDelimiter string `toml:"csv_delimiter" json:"csv_delimiter"` + CSVHeaderRowCount int `toml:"csv_header_row_count" json:"csv_header_row_count"` + CSVMeasurementColumn string `toml:"csv_measurement_column" json:"csv_measurement_column"` + CSVSkipColumns int `toml:"csv_skip_columns" json:"csv_skip_columns"` + CSVSkipRows int `toml:"csv_skip_rows" json:"csv_skip_rows"` + CSVTagColumns []string `toml:"csv_tag_columns" json:"csv_tag_columns"` + CSVTimestampColumn string `toml:"csv_timestamp_column" json:"csv_timestamp_column"` + CSVTimestampFormat string `toml:"csv_timestamp_format" json:"csv_timestamp_format"` + CSVTimezone string `toml:"csv_timezone" json:"csv_timezone"` + CSVTrimSpace bool `toml:"csv_trim_space" json:"csv_trim_space"` + CSVSkipValues []string `toml:"csv_skip_values" json:"csv_skip_values"` // FormData configuration - FormUrlencodedTagKeys []string `toml:"form_urlencoded_tag_keys"` + FormUrlencodedTagKeys []string `toml:"form_urlencoded_tag_keys" json:"form_urlencoded_tag_keys"` // Value configuration - ValueFieldName string `toml:"value_field_name"` + ValueFieldName string `toml:"value_field_name" json:"value_field_name"` // XPath configuration - XPathPrintDocument bool `toml:"xpath_print_document"` - XPathProtobufFile string `toml:"xpath_protobuf_file"` - XPathProtobufType string `toml:"xpath_protobuf_type"` + XPathPrintDocument bool `toml:"xpath_print_document" json:"xpath_print_document"` + XPathProtobufFile string `toml:"xpath_protobuf_file" json:"xpath_protobuf_file"` + XPathProtobufType string `toml:"xpath_protobuf_type" json:"xpath_protobuf_type"` XPathConfig []XPathConfig // JSONPath configuration - JSONV2Config []JSONV2Config `toml:"json_v2"` + JSONV2Config []JSONV2Config `toml:"json_v2" json:"json_v2"` } type XPathConfig xpath.Config diff --git a/plugins/processors/aws/ec2/ec2.go b/plugins/processors/aws/ec2/ec2.go index 7126214152a51..088ec09c83f5f 100644 --- a/plugins/processors/aws/ec2/ec2.go +++ b/plugins/processors/aws/ec2/ec2.go @@ -124,6 +124,20 @@ func (r *AwsEc2Processor) Init() error { return errors.New("no tags specified in configuration") } + for _, tag := range r.ImdsTags { + if len(tag) == 0 || !isImdsTagAllowed(tag) { + return fmt.Errorf("not allowed metadata tag specified in configuration: %s", tag) + } + r.imdsTags[tag] = struct{}{} + } + if len(r.imdsTags) == 0 && len(r.EC2Tags) == 0 { + return errors.New("no allowed metadata tags specified in configuration") + } + + return nil +} + +func (r *AwsEc2Processor) Start(acc telegraf.Accumulator) error { ctx := context.Background() cfg, err := awsconfig.LoadDefaultConfig(ctx) if err != nil { @@ -161,21 +175,6 @@ func (r *AwsEc2Processor) Init() error { } } - for _, tag := range r.ImdsTags { - if len(tag) > 0 && isImdsTagAllowed(tag) { - r.imdsTags[tag] = struct{}{} - } else { - return fmt.Errorf("not allowed metadata tag specified in configuration: %s", tag) - } - } - if len(r.imdsTags) == 0 && len(r.EC2Tags) == 0 { - return errors.New("no allowed metadata tags specified in configuration") - } - - return nil -} - -func (r *AwsEc2Processor) Start(acc telegraf.Accumulator) error { if r.Ordered { r.parallel = parallel.NewOrdered(acc, r.asyncAdd, DefaultMaxOrderedQueueSize, r.MaxParallelCalls) } else { diff --git a/plugins/processors/aws/ec2/ec2_test.go b/plugins/processors/aws/ec2/ec2_test.go index 8eb599206ff99..e7f16a01252e1 100644 --- a/plugins/processors/aws/ec2/ec2_test.go +++ b/plugins/processors/aws/ec2/ec2_test.go @@ -1,8 +1,10 @@ package ec2 import ( + "context" "testing" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" @@ -13,8 +15,7 @@ func TestBasicStartup(t *testing.T) { p.Log = &testutil.Logger{} p.ImdsTags = []string{"accountId", "instanceId"} acc := &testutil.Accumulator{} - require.NoError(t, p.Start(acc)) - require.NoError(t, p.Stop()) + require.NoError(t, p.Init()) require.Len(t, acc.GetTelegrafMetrics(), 0) require.Len(t, acc.Errors, 0) @@ -26,8 +27,7 @@ func TestBasicStartupWithEC2Tags(t *testing.T) { p.ImdsTags = []string{"accountId", "instanceId"} p.EC2Tags = []string{"Name"} acc := &testutil.Accumulator{} - require.NoError(t, p.Start(acc)) - require.NoError(t, p.Stop()) + require.NoError(t, p.Init()) require.Len(t, acc.GetTelegrafMetrics(), 0) require.Len(t, acc.Errors, 0) @@ -50,10 +50,18 @@ func TestBasicInitInvalidTagsReturnAnError(t *testing.T) { } func TestLoadingConfig(t *testing.T) { - confFile := []byte("[[processors.aws_ec2]]" + "\n" + sampleConfig) c := config.NewConfig() - err := c.LoadConfigData(confFile) - require.NoError(t, err) + c.SetAgent(&agenthelper.TestAgentController{}) + err := c.LoadConfigData(context.Background(), context.Background(), []byte( + ` + [[processors.aws_ec2]] + imds_tags = ["availabilityZone"] + ec2_tags = ["availabilityZone"] + timeout = "30s" + max_parallel_calls = 10 + `, + )) - require.Len(t, c.Processors, 1) + require.NoError(t, err) + require.Len(t, c.Processors(), 1) } diff --git a/plugins/processors/reverse_dns/reversedns_test.go b/plugins/processors/reverse_dns/reversedns_test.go index 5fcce5fb4725a..eab2853251b1e 100644 --- a/plugins/processors/reverse_dns/reversedns_test.go +++ b/plugins/processors/reverse_dns/reversedns_test.go @@ -1,12 +1,14 @@ package reverse_dns import ( + "context" "runtime" "testing" "time" "github.com/stretchr/testify/require" + "github.com/influxdata/telegraf/agent" "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/metric" "github.com/influxdata/telegraf/testutil" @@ -54,8 +56,11 @@ func TestSimpleReverseLookup(t *testing.T) { func TestLoadingConfig(t *testing.T) { c := config.NewConfig() - err := c.LoadConfigData([]byte("[[processors.reverse_dns]]\n" + sampleConfig)) + ctx := context.Background() + a := agent.NewAgent(ctx, c) + c.SetAgent(a) + err := c.LoadConfigData(ctx, ctx, []byte("[[processors.reverse_dns]]\n"+sampleConfig)) require.NoError(t, err) - require.Len(t, c.Processors, 1) + require.Len(t, c.Processors(), 1) } diff --git a/plugins/processors/starlark/starlark_test.go b/plugins/processors/starlark/starlark_test.go index 15152a2f349c3..16609111fa67b 100644 --- a/plugins/processors/starlark/starlark_test.go +++ b/plugins/processors/starlark/starlark_test.go @@ -1,6 +1,7 @@ package starlark import ( + "context" "errors" "fmt" "io/ioutil" @@ -11,7 +12,9 @@ import ( "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/agenthelper" "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/models" "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" @@ -2604,14 +2607,15 @@ def apply(metric): // Build a Starlark plugin from the provided configuration. func buildPlugin(configContent string) (*Starlark, error) { c := config.NewConfig() - err := c.LoadConfigData([]byte(configContent)) + c.SetAgent(&agenthelper.TestAgentController{}) + err := c.LoadConfigData(context.Background(), context.Background(), []byte(configContent)) if err != nil { return nil, err } - if len(c.Processors) != 1 { + if len(c.Processors()) != 1 { return nil, errors.New("Only one processor was expected") } - plugin, ok := (c.Processors[0].Processor).(*Starlark) + plugin, ok := (c.Processors()[0].(*models.RunningProcessor).Processor).(*Starlark) if !ok { return nil, errors.New("Only a Starlark processor was expected") } @@ -3189,7 +3193,7 @@ func TestAllScriptTestData(t *testing.T) { paths := []string{"testdata", "plugins/processors/starlark/testdata"} for _, testdataPath := range paths { filepath.Walk(testdataPath, func(path string, info os.FileInfo, err error) error { - if info == nil || info.IsDir() { + if info == nil || info.IsDir() || !strings.HasSuffix(info.Name(), ".star") { return nil } fn := path diff --git a/plugins/storage/all/all.go b/plugins/storage/all/all.go new file mode 100644 index 0000000000000..a5313df81458b --- /dev/null +++ b/plugins/storage/all/all.go @@ -0,0 +1,7 @@ +package all + +import ( + // Blank imports for plugins to register themselves + _ "github.com/influxdata/telegraf/plugins/storage/boltdb" + _ "github.com/influxdata/telegraf/plugins/storage/jsonfile" +) diff --git a/plugins/storage/boltdb/boltdb.go b/plugins/storage/boltdb/boltdb.go new file mode 100644 index 0000000000000..a0240b0e411d5 --- /dev/null +++ b/plugins/storage/boltdb/boltdb.go @@ -0,0 +1,84 @@ +package boltdb + +import ( + "fmt" + + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/storage" + "github.com/ugorji/go/codec" + bolt "go.etcd.io/bbolt" +) + +var ( + codecHandle = &codec.MsgpackHandle{} +) + +type BoltDBStorage struct { + Filename string `toml:"file"` + + db *bolt.DB +} + +func (s *BoltDBStorage) Init() error { + if len(s.Filename) == 0 { + return fmt.Errorf("Storage service requires filename of db") + } + db, err := bolt.Open(s.Filename, 0600, nil) + if err != nil { + return fmt.Errorf("couldn't open file %q: %w", s.Filename, err) + } + s.db = db + return nil +} + +func (s *BoltDBStorage) Close() error { + return s.db.Close() +} + +func (s *BoltDBStorage) Load(namespace, key string, obj interface{}) error { + err := s.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(namespace)) + if b == nil { + // don't error on not found + return nil + } + v := b.Get([]byte(key)) + decoder := codec.NewDecoderBytes(v, codecHandle) + + if err := decoder.Decode(obj); err != nil { + return fmt.Errorf("decoding: %w", err) + } + + return nil + }) + return err +} + +func (s *BoltDBStorage) Save(namespace, key string, value interface{}) error { + return s.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(namespace)) + if b == nil { + bucket, err := tx.CreateBucket([]byte(namespace)) + if err != nil { + return err + } + b = bucket + } + var byt []byte + enc := codec.NewEncoderBytes(&byt, codecHandle) + if err := enc.Encode(value); err != nil { + return fmt.Errorf("encoding: %w", err) + } + return b.Put([]byte(key), byt) + }) +} + +func (s *BoltDBStorage) GetName() string { + return "internal" +} + +func init() { + storage.Add("internal", func() config.StoragePlugin { + return &BoltDBStorage{} + }) +} diff --git a/plugins/storage/boltdb/boltdb_test.go b/plugins/storage/boltdb/boltdb_test.go new file mode 100644 index 0000000000000..0d02deac7af03 --- /dev/null +++ b/plugins/storage/boltdb/boltdb_test.go @@ -0,0 +1,48 @@ +package boltdb + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +type Foo struct { + Shoes int + Carrots string + Aardvark bool + Neon float64 +} + +func TestSaveLoadCycle(t *testing.T) { + p := &BoltDBStorage{ + Filename: "testdb.db", + } + defer func() { + _ = os.Remove("testdb.db") + }() + err := p.Init() + require.NoError(t, err) + + // check that loading a missing key doesn't fail + err = p.Load("testing", "foo", &Foo{}) + require.NoError(t, err) + + foo := Foo{ + Shoes: 3, + Carrots: "blue", + Aardvark: true, + Neon: 3.1415, + } + err = p.Save("testing", "foo", foo) + require.NoError(t, err) + + obj := &Foo{} + err = p.Load("testing", "foo", obj) + require.NoError(t, err) + + require.EqualValues(t, foo, *obj) + + err = p.Close() + require.NoError(t, err) +} diff --git a/plugins/storage/jsonfile/jsonfile.go b/plugins/storage/jsonfile/jsonfile.go new file mode 100644 index 0000000000000..8a78482a7fd19 --- /dev/null +++ b/plugins/storage/jsonfile/jsonfile.go @@ -0,0 +1,79 @@ +package jsonfile + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/storage" +) + +type JSONFileStorage struct { + Filename string `toml:"file"` +} + +func (s *JSONFileStorage) Init() error { + if len(s.Filename) == 0 { + return fmt.Errorf("Storage service requires filename") + } + return nil +} + +func (s *JSONFileStorage) Close() error { + return nil +} + +func (s *JSONFileStorage) Load(namespace, key string, obj interface{}) error { + f, err := os.Open(s.Filename) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + dec := json.NewDecoder(f) + m := map[string]interface{}{} + err = dec.Decode(&m) + if err != nil { + return err + } + if v, ok := m[namespace]; ok { + m = v.(map[string]interface{}) + } + if v, ok := m[key]; ok { + b, err := json.Marshal(v) + if err != nil { + return err + } + err = json.Unmarshal(b, obj) + if err != nil { + return err + } + + return nil + } + return nil +} + +func (s *JSONFileStorage) Save(namespace, key string, value interface{}) error { + m := map[string]interface{}{} + m[namespace] = map[string]interface{}{ + key: value, + } + data, err := json.Marshal(m) + if err != nil { + return err + } + return os.WriteFile(s.Filename, data, 0600) +} + +func (s *JSONFileStorage) GetName() string { + return "jsonfile" +} + +func init() { + storage.Add("jsonfile", func() config.StoragePlugin { + return &JSONFileStorage{} + }) +} diff --git a/plugins/storage/registry.go b/plugins/storage/registry.go new file mode 100644 index 0000000000000..b8401a886932f --- /dev/null +++ b/plugins/storage/registry.go @@ -0,0 +1,7 @@ +package storage + +import "github.com/influxdata/telegraf/config" + +func Add(name string, creator config.StorageCreator) { + config.StoragePlugins[name] = creator +} diff --git a/testutil/accumulator.go b/testutil/accumulator.go index 4da3a76fcc8ca..16e412ed1b665 100644 --- a/testutil/accumulator.go +++ b/testutil/accumulator.go @@ -200,6 +200,10 @@ func (a *Accumulator) WithTracking(_ int) telegraf.TrackingAccumulator { return a } +func (a *Accumulator) WithNewMetricMaker(logName string, logger telegraf.Logger, f func(metric telegraf.Metric) telegraf.Metric) telegraf.Accumulator { + return a +} + func (a *Accumulator) AddTrackingMetric(m telegraf.Metric) telegraf.TrackingID { a.AddMetric(m) return newTrackingID() @@ -739,3 +743,6 @@ func (n *NopAccumulator) AddMetric(telegraf.Metric) {} func (n *NopAccumulator) SetPrecision(_ time.Duration) {} func (n *NopAccumulator) AddError(_ error) {} func (n *NopAccumulator) WithTracking(_ int) telegraf.TrackingAccumulator { return nil } +func (n *NopAccumulator) WithNewMetricMaker(logName string, logger telegraf.Logger, f func(metric telegraf.Metric) telegraf.Metric) telegraf.Accumulator { + return nil +}