From a1b64cdbe6945c8bad00e8fd8132765cb5f6a749 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Mon, 14 Aug 2017 16:00:41 +0200 Subject: [PATCH] Remove all references to removed output.X.flush_interval settings (#4880) * Remove all references to flush_interval The outputs setting flush_interval has been removed in the past. Remove this setting from reference configuration and docs as well. * Fail if removed flush_interval is configured --- CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 8 -------- filebeat/filebeat.reference.yml | 8 -------- heartbeat/heartbeat.reference.yml | 8 -------- libbeat/_meta/config.reference.yml | 8 -------- libbeat/docs/outputconfig.asciidoc | 10 ---------- libbeat/outputs/fileout/file.go | 1 - libbeat/outputs/logstash/logstash_integration_test.go | 2 -- libbeat/outputs/output_reg.go | 5 +++++ libbeat/tests/system/test_base.py | 1 - metricbeat/metricbeat.reference.yml | 8 -------- packetbeat/packetbeat.reference.yml | 8 -------- winlogbeat/winlogbeat.reference.yml | 8 -------- 13 files changed, 6 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 89bda67a23b..5694d20f952 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di *Affecting all Beats* - The log directory (`path.log`) for Windows services is now set to `C:\ProgramData\[beatname]\logs`. {issue}4764[4764] +- Fail if removed setting output.X.flush_interval is explicitly configured. *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index ef9441b418a..049ba139cac 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -236,11 +236,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -440,9 +435,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index dcd7d03653b..37c52fa679e 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -641,11 +641,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -845,9 +840,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index ba14c3841bf..c3b0ebdce58 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -385,11 +385,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -589,9 +584,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 294c3b93039..478a48e97cc 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -171,11 +171,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -375,9 +370,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 751dd55650a..61ba17df75a 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -292,12 +292,6 @@ spooler size. The http request timeout in seconds for the Elasticsearch request. The default is 90. -===== `flush_interval` - -The number of seconds to wait for new events between two bulk API index requests. -If `bulk_max_size` is reached before this interval expires, additional bulk index -requests are made. - ===== `ssl` Configuration options for SSL parameters like the certificate authority to use @@ -731,10 +725,6 @@ The ACK reliability level required from broker. 0=no response, 1=wait for local Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error. -===== `flush_interval` - -The number of seconds to wait for new events between two producer API calls. - ===== `ssl` Configuration options for SSL parameters like the root CA for Kafka connections. See diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index a0f0529b564..faa9cc79d7c 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -32,7 +32,6 @@ func makeFileout( } // disable bulk support in publisher pipeline - cfg.SetInt("flush_interval", -1, -1) cfg.SetInt("bulk_max_size", -1, -1) fo := &fileOutput{beat: beat, stats: stats} diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index fa6e553ae19..e69fb050b9b 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -148,12 +148,10 @@ func newTestElasticsearchOutput(t *testing.T, test string) *testOutputer { index := testElasticsearchIndex(test) connection := esConnect(t, index) - flushInterval := 0 bulkSize := 0 config, _ := common.NewConfigFrom(map[string]interface{}{ "hosts": []string{getElasticsearchHost()}, "index": connection.index, - "flush_interval": &flushInterval, "bulk_max_size": &bulkSize, "username": os.Getenv("ES_USER"), "password": os.Getenv("ES_PASS"), diff --git a/libbeat/outputs/output_reg.go b/libbeat/outputs/output_reg.go index 7d2782166c8..207d3ae5f6e 100644 --- a/libbeat/outputs/output_reg.go +++ b/libbeat/outputs/output_reg.go @@ -5,6 +5,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" ) var outputReg = map[string]Factory{} @@ -43,5 +44,9 @@ func Load(info beat.Info, stats *Stats, name string, config *common.Config) (Gro return Group{}, fmt.Errorf("output type %v undefined", name) } + if err := cfgwarn.CheckRemoved5xSetting(config, "flush_interval"); err != nil { + return Fail(err) + } + return factory(info, stats, config) } diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py index 0bb8ccd1d96..c328500a57b 100644 --- a/libbeat/tests/system/test_base.py +++ b/libbeat/tests/system/test_base.py @@ -154,7 +154,6 @@ def test_console_output_size_flush(self): console={ "pretty": "false", "bulk_max_size": 1, - "flush_interval": "1h" } ) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index ef8404cba83..dd6811e7b1b 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -605,11 +605,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -809,9 +804,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index f2e44f01ba2..58e72b6228c 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -623,11 +623,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -827,9 +822,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 6c3bfac90cc..668b3deac40 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -200,11 +200,6 @@ output.elasticsearch: # Configure http request timeout before failing an request to Elasticsearch. #timeout: 90 - # The number of seconds to wait for new events between two bulk API index requests. - # If `bulk_max_size` is reached before this interval expires, addition bulk index - # requests are made. - #flush_interval: 1s - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true @@ -404,9 +399,6 @@ output.elasticsearch: # on error. #required_acks: 1 - # The number of seconds to wait for new events between two producer API calls. - #flush_interval: 1s - # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats