diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 35f7bbeadfa2..a9ba19701abb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -82,6 +82,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove all alias fields pointing to ECS fields from modules. This affects the Suricata and Traefik modules. {issue}10535[10535] {pull}26627[26627] *Heartbeat* +- Remove long deprecated `watch_poll` functionality. {pull}27166[27166] *Journalbeat* diff --git a/heartbeat/_meta/config/beat.reference.yml.tmpl b/heartbeat/_meta/config/beat.reference.yml.tmpl index b097bd94b653..d4ad845228aa 100644 --- a/heartbeat/_meta/config/beat.reference.yml.tmpl +++ b/heartbeat/_meta/config/beat.reference.yml.tmpl @@ -71,26 +71,6 @@ heartbeat.monitors: # sub-dictionary. Default is false. #fields_under_root: false - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - - # The Ingest Node pipeline ID associated with this input. If this is set, it - # overwrites the pipeline option from the Elasticsearch output. - #pipeline: - - # The index name associated with this input. If this is set, it - # overwrites the index option from the Elasticsearch output. - #index: - - # Set to true to publish fields with null values in events. - #keep_null: false - # Define a directory to load monitor definitions from. Definitions take the form # of individual yaml files. # heartbeat.config.monitors: @@ -168,15 +148,6 @@ heartbeat.monitors: # Required TLS protocols #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # The Ingest Node pipeline ID associated with this input. If this is set, it # overwrites the pipeline option from the Elasticsearch output. #pipeline: @@ -210,14 +181,6 @@ heartbeat.monitors: ipv6: true mode: any - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # Optional HTTP proxy url. #proxy_url: '' @@ -267,16 +230,6 @@ heartbeat.monitors: # equals: # myField: expectedValue - - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # The Ingest Node pipeline ID associated with this input. If this is set, it # overwrites the pipeline option from the Elasticsearch output. #pipeline: diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go index 85bfb79fe207..080ce347f234 100644 --- a/heartbeat/beater/heartbeat.go +++ b/heartbeat/beater/heartbeat.go @@ -73,7 +73,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) { config: parsedConfig, scheduler: scheduler, // dynamicFactory is the factory used for dynamic configs, e.g. autodiscover / reload - dynamicFactory: monitors.NewFactory(b.Info, scheduler, false), + dynamicFactory: monitors.NewFactory(b.Info, scheduler), } return bt, nil } @@ -125,7 +125,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { // RunStaticMonitors runs the `heartbeat.monitors` portion of the yaml config if present. func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) (stop func(), err error) { - factory := monitors.NewFactory(b.Info, bt.scheduler, true) + factory := monitors.NewFactory(b.Info, bt.scheduler) var runners []cfgfile.Runner for _, cfg := range bt.config.Monitors { diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index c2c74fca2f16..14597f2a27c5 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -71,26 +71,6 @@ heartbeat.monitors: # sub-dictionary. Default is false. #fields_under_root: false - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - - # The Ingest Node pipeline ID associated with this input. If this is set, it - # overwrites the pipeline option from the Elasticsearch output. - #pipeline: - - # The index name associated with this input. If this is set, it - # overwrites the index option from the Elasticsearch output. - #index: - - # Set to true to publish fields with null values in events. - #keep_null: false - # Define a directory to load monitor definitions from. Definitions take the form # of individual yaml files. # heartbeat.config.monitors: @@ -168,15 +148,6 @@ heartbeat.monitors: # Required TLS protocols #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # The Ingest Node pipeline ID associated with this input. If this is set, it # overwrites the pipeline option from the Elasticsearch output. #pipeline: @@ -210,14 +181,6 @@ heartbeat.monitors: ipv6: true mode: any - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # Optional HTTP proxy url. #proxy_url: '' @@ -267,16 +230,6 @@ heartbeat.monitors: # equals: # myField: expectedValue - - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # The Ingest Node pipeline ID associated with this input. If this is set, it # overwrites the pipeline option from the Elasticsearch output. #pipeline: diff --git a/heartbeat/monitors/factory.go b/heartbeat/monitors/factory.go index 78d5aa648095..5e82969fd18d 100644 --- a/heartbeat/monitors/factory.go +++ b/heartbeat/monitors/factory.go @@ -34,9 +34,8 @@ import ( // RunnerFactory that can be used to create cfg.Runner cast versions of Monitor // suitable for config reloading. type RunnerFactory struct { - info beat.Info - sched *scheduler.Scheduler - allowWatches bool + info beat.Info + sched *scheduler.Scheduler } type publishSettings struct { @@ -59,8 +58,8 @@ type publishSettings struct { } // NewFactory takes a scheduler and creates a RunnerFactory that can create cfgfile.Runner(Monitor) objects. -func NewFactory(info beat.Info, sched *scheduler.Scheduler, allowWatches bool) *RunnerFactory { - return &RunnerFactory{info, sched, allowWatches} +func NewFactory(info beat.Info, sched *scheduler.Scheduler) *RunnerFactory { + return &RunnerFactory{info, sched} } // Create makes a new Runner for a new monitor with the given Config. @@ -76,13 +75,13 @@ func (f *RunnerFactory) Create(p beat.Pipeline, c *common.Config) (cfgfile.Runne } p = pipetool.WithClientConfigEdit(p, configEditor) - monitor, err := newMonitor(c, plugin.GlobalPluginsReg, p, f.sched, f.allowWatches) + monitor, err := newMonitor(c, plugin.GlobalPluginsReg, p, f.sched) return monitor, err } // CheckConfig checks to see if the given monitor config is valid. func (f *RunnerFactory) CheckConfig(config *common.Config) error { - return checkMonitorConfig(config, plugin.GlobalPluginsReg, f.allowWatches) + return checkMonitorConfig(config, plugin.GlobalPluginsReg) } func newCommonPublishConfigs(info beat.Info, cfg *common.Config) (pipetool.ConfigEditor, error) { diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go index 4003cacdf422..6f976235ee35 100644 --- a/heartbeat/monitors/monitor.go +++ b/heartbeat/monitors/monitor.go @@ -18,8 +18,6 @@ package monitors import ( - "bytes" - "encoding/json" "fmt" "sync" @@ -31,7 +29,6 @@ import ( "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" "github.com/elastic/beats/v7/heartbeat/scheduler" - "github.com/elastic/beats/v7/heartbeat/watcher" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -55,10 +52,6 @@ type Monitor struct { internalsMtx sync.Mutex close func() error - // Watch related fields - watchPollTasks []*configuredJob - watch watcher.Watch - pipelineConnector beat.PipelineConnector // stats is the countersRecorder used to record lifecycle events @@ -72,17 +65,14 @@ func (m *Monitor) String() string { return fmt.Sprintf("Monitor", m.stdFields.Name, m.enabled) } -func checkMonitorConfig(config *common.Config, registrar *plugin.PluginsReg, allowWatches bool) error { - m, err := newMonitor(config, registrar, nil, nil, allowWatches) +func checkMonitorConfig(config *common.Config, registrar *plugin.PluginsReg) error { + m, err := newMonitor(config, registrar, nil, nil) if m != nil { m.Stop() // Stop the monitor to free up the ID from uniqueness checks } return err } -// ErrWatchesDisabled is returned when the user attempts to declare a watch poll file in a -var ErrWatchesDisabled = errors.New("watch poll files are only allowed in heartbeat.yml, not dynamic configs") - // uniqueMonitorIDs is used to keep track of explicitly configured monitor IDs and ensure no duplication within a // given heartbeat instance. var uniqueMonitorIDs sync.Map @@ -100,9 +90,8 @@ func newMonitor( registrar *plugin.PluginsReg, pipelineConnector beat.PipelineConnector, scheduler *scheduler.Scheduler, - allowWatches bool, ) (*Monitor, error) { - m, err := newMonitorUnsafe(config, registrar, pipelineConnector, scheduler, allowWatches) + m, err := newMonitorUnsafe(config, registrar, pipelineConnector, scheduler) if m != nil && err != nil { m.Stop() } @@ -116,7 +105,6 @@ func newMonitorUnsafe( registrar *plugin.PluginsReg, pipelineConnector beat.PipelineConnector, scheduler *scheduler.Scheduler, - allowWatches bool, ) (*Monitor, error) { // Extract just the Id, Type, and Enabled fields from the config // We'll parse things more precisely later once we know what exact type of @@ -137,7 +125,6 @@ func newMonitorUnsafe( scheduler: scheduler, configuredJobs: []*configuredJob{}, pipelineConnector: pipelineConnector, - watchPollTasks: []*configuredJob{}, internalsMtx: sync.Mutex{}, config: config, stats: pluginFactory.Stats, @@ -171,20 +158,6 @@ func newMonitorUnsafe( return m, err } - err = m.makeWatchTasks(pluginFactory) - if err != nil { - return m, err - } - - if len(m.watchPollTasks) > 0 { - if !allowWatches { - return m, ErrWatchesDisabled - } - - logp.Info(`Obsolete option 'watch.poll_file' declared. This will be removed in a future release. -See https://www.elastic.co/guide/en/beats/heartbeat/current/configuration-heartbeat-options.html for more info`) - } - return m, nil } @@ -227,74 +200,6 @@ func (m *Monitor) makeTasks(config *common.Config, jobs []jobs.Job) ([]*configur return mTasks, nil } -func (m *Monitor) makeWatchTasks(pluginFactory plugin.PluginFactory) error { - watchCfg := watcher.DefaultWatchConfig - err := m.config.Unpack(&watchCfg) - if err != nil { - return err - } - - if len(watchCfg.Path) > 0 { - m.watch, err = watcher.NewFilePoller(watchCfg.Path, watchCfg.Poll, func(content []byte) { - var newTasks []*configuredJob - - dec := json.NewDecoder(bytes.NewBuffer(content)) - for dec.More() { - var obj map[string]interface{} - err = dec.Decode(&obj) - if err != nil { - logp.Err("Failed parsing JSON object: %v", err) - return - } - - cfg, err := common.NewConfigFrom(obj) - if err != nil { - logp.Err("Failed normalizing JSON input: %v", err) - return - } - - merged, err := common.MergeConfigs(m.config, cfg) - if err != nil { - logp.Err("Could not merge config: %v", err) - return - } - - p, err := pluginFactory.Create(merged) - m.close = p.Close - m.endpoints = p.Endpoints - if err != nil { - logp.Err("Could not create job from watch file: %v", err) - } - - watchTasks, err := m.makeTasks(merged, p.Jobs) - if err != nil { - logp.Err("Could not make configuredJob for config: %v", err) - return - } - - newTasks = append(newTasks, watchTasks...) - } - - m.internalsMtx.Lock() - defer m.internalsMtx.Unlock() - - for _, t := range m.watchPollTasks { - t.Stop() - } - m.watchPollTasks = newTasks - for _, t := range m.watchPollTasks { - t.Start() - } - }) - - if err != nil { - return err - } - } - - return nil -} - // Start starts the monitor's execution using its configured scheduler. func (m *Monitor) Start() { m.internalsMtx.Lock() @@ -304,10 +209,6 @@ func (m *Monitor) Start() { t.Start() } - for _, t := range m.watchPollTasks { - t.Start() - } - m.stats.StartMonitor(int64(m.endpoints)) } @@ -322,10 +223,6 @@ func (m *Monitor) Stop() { t.Stop() } - for _, t := range m.watchPollTasks { - t.Stop() - } - if m.close != nil { err := m.close() if err != nil { diff --git a/heartbeat/monitors/monitor_test.go b/heartbeat/monitors/monitor_test.go index b0d02f819f6b..0f00828bf9b2 100644 --- a/heartbeat/monitors/monitor_test.go +++ b/heartbeat/monitors/monitor_test.go @@ -39,7 +39,7 @@ func TestMonitor(t *testing.T) { require.NoError(t, err) defer sched.Stop() - mon, err := newMonitor(serverMonConf, reg, pipelineConnector, sched, false) + mon, err := newMonitor(serverMonConf, reg, pipelineConnector, sched) require.NoError(t, err) mon.Start() @@ -90,11 +90,11 @@ func TestDuplicateMonitorIDs(t *testing.T) { defer sched.Stop() makeTestMon := func() (*Monitor, error) { - return newMonitor(serverMonConf, reg, pipelineConnector, sched, false) + return newMonitor(serverMonConf, reg, pipelineConnector, sched) } // Ensure that an error is returned on a bad config - _, m0Err := newMonitor(badConf, reg, pipelineConnector, sched, false) + _, m0Err := newMonitor(badConf, reg, pipelineConnector, sched) require.Error(t, m0Err) // Would fail if the previous newMonitor didn't free the monitor.id @@ -125,7 +125,7 @@ func TestCheckInvalidConfig(t *testing.T) { require.NoError(t, err) defer sched.Stop() - m, err := newMonitor(serverMonConf, reg, pipelineConnector, sched, false) + m, err := newMonitor(serverMonConf, reg, pipelineConnector, sched) // This could change if we decide the contract for newMonitor should always return a monitor require.Nil(t, m, "For this test to work we need a nil value for the monitor.") @@ -133,5 +133,5 @@ func TestCheckInvalidConfig(t *testing.T) { require.Equal(t, 0, built.Load()) require.Equal(t, 0, closed.Load()) - require.Error(t, checkMonitorConfig(serverMonConf, reg, false)) + require.Error(t, checkMonitorConfig(serverMonConf, reg)) } diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index c2c74fca2f16..14597f2a27c5 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -71,26 +71,6 @@ heartbeat.monitors: # sub-dictionary. Default is false. #fields_under_root: false - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - - # The Ingest Node pipeline ID associated with this input. If this is set, it - # overwrites the pipeline option from the Elasticsearch output. - #pipeline: - - # The index name associated with this input. If this is set, it - # overwrites the index option from the Elasticsearch output. - #index: - - # Set to true to publish fields with null values in events. - #keep_null: false - # Define a directory to load monitor definitions from. Definitions take the form # of individual yaml files. # heartbeat.config.monitors: @@ -168,15 +148,6 @@ heartbeat.monitors: # Required TLS protocols #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # The Ingest Node pipeline ID associated with this input. If this is set, it # overwrites the pipeline option from the Elasticsearch output. #pipeline: @@ -210,14 +181,6 @@ heartbeat.monitors: ipv6: true mode: any - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # Optional HTTP proxy url. #proxy_url: '' @@ -267,16 +230,6 @@ heartbeat.monitors: # equals: # myField: expectedValue - - # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE - # Configure file json file to be watched for changes to the monitor: - #watch.poll_file: - # Path to check for updates. - #path: - - # Interval between file file changed checks. - #interval: 5s - # The Ingest Node pipeline ID associated with this input. If this is set, it # overwrites the pipeline option from the Elasticsearch output. #pipeline: