Remove prospector support in Filebeat (elastic#8909)

## Add CheckRemoved6xSettings and CheckRemoved6xSettings Add new methods to verify obsolete configuration for 7.0, added missing test for them. ## Remove the prospector option in the configuration. In 6.3 we have deprecated the usage of the 'prospector' options, in 7.0 its now obsolete. This commit does the following: - Remove any backward compatibility fixes. - Add warning when using the prospector key in configuration. - Remove any usage of prospector in tests. - Remove any usage or prospector in module expectation files. - Adjust any integration tests to not check for prospector.type. - Remove the prospector type from the fields. - Remove any shims created by the prospector package.
efd6 · Nov 6, 2018 · 755831e · 755831e
1 parent b14f42b
commit 755831e
Show file tree

Hide file tree

Showing 71 changed files with 346 additions and 642 deletions.
diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc
@@ -63,3 +63,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Add `mage.KibanaDashboards` for collecting Kibana dashboards and generating index patterns. {pull}8615[8615]
 - Allow to disable config resolver using the `Settings.DisableConfigResolver` field when initializing libbeat. {pull}8769[8769]
 - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889]
+- Add `cfgwarn.CheckRemoved6xSetting(s)` to display a warning for options removed in 7.0. {pull}8909[8909]
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -21,6 +21,8 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 
 *Filebeat*
 
+- Remove the deprecated `prospector(s)` option in the configuration use `input(s)` instead. {pull}8909[8909]
+
 *Heartbeat*
 
 *Journalbeat*

diff --git a/filebeat/Makefile b/filebeat/Makefile
@@ -34,7 +34,7 @@ collect-docs: python-env
 	@mkdir -p docs/modules
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/filebeat/scripts/docs_collector.py --beat ${BEAT_NAME}
 
-# Generate imports for prospectors
+# Generate imports for inputs
 .PHONY: imports
 imports: python-env
 	@mkdir -p include

diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml
@@ -29,13 +29,6 @@
       description: >
         Log stream when reading container logs, can be 'stdout' or 'stderr'
 
-    - name: prospector.type
-      required: true
-      deprecated: 6.3
-      description: >
-        The input type from which the event was generated. This field is set to the value specified
-        for the `type` option in the input section of the Filebeat config file. (DEPRECATED: see `input.type`)
-
     - name: input.type
       required: true
       description: >

diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go
@@ -77,20 +77,8 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 		return nil, err
 	}
 
-	if len(config.Prospectors) > 0 {
-		cfgwarn.Deprecate("7.0.0", "prospectors are deprecated, Use `inputs` instead.")
-		if len(config.Inputs) > 0 {
-			return nil, fmt.Errorf("prospectors and inputs used in the configuration file, define only inputs not both")
-		}
-		config.Inputs = config.Prospectors
-	}
-
-	if config.ConfigProspector != nil {
-		cfgwarn.Deprecate("7.0.0", "config.prospectors are deprecated, Use `config.inputs` instead.")
-		if config.ConfigInput != nil {
-			return nil, fmt.Errorf("config.prospectors and config.inputs used in the configuration file, define only config.inputs not both")
-		}
-		config.ConfigInput = config.ConfigProspector
+	if err := cfgwarn.CheckRemoved6xSettings(rawConfig, "prospectors", "config.prospectors"); err != nil {
+		return nil, err
 	}
 
 	moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info.Version, true)

diff --git a/filebeat/channel/factory.go b/filebeat/channel/factory.go
@@ -110,9 +110,6 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c
 		}
 	}
 	if config.Type != "" {
-		fields["prospector"] = common.MapStr{
-			"type": config.Type,
-		}
 		fields["input"] = common.MapStr{
 			"type": config.Type,
 		}

diff --git a/filebeat/config/config.go b/filebeat/config/config.go
@@ -40,15 +40,13 @@ const (
 
 type Config struct {
 	Inputs                  []*common.Config     `config:"inputs"`
-	Prospectors             []*common.Config     `config:"prospectors"`
 	RegistryFile            string               `config:"registry_file"`
 	RegistryFilePermissions os.FileMode          `config:"registry_file_permissions"`
 	RegistryFlush           time.Duration        `config:"registry_flush"`
 	ConfigDir               string               `config:"config_dir"`
 	ShutdownTimeout         time.Duration        `config:"shutdown_timeout"`
 	Modules                 []*common.Config     `config:"modules"`
 	ConfigInput             *common.Config       `config:"config.inputs"`
-	ConfigProspector        *common.Config       `config:"config.prospectors"`
 	ConfigModules           *common.Config       `config:"config.modules"`
 	Autodiscover            *autodiscover.Config `config:"autodiscover"`
 	OverwritePipelines      bool                 `config:"overwrite_pipelines"`
@@ -106,14 +104,6 @@ func mergeConfigFiles(configFiles []string, config *Config) error {
 			return fmt.Errorf("Failed to read %s: %s", file, err)
 		}
 
-		if len(tmpConfig.Filebeat.Prospectors) > 0 {
-			cfgwarn.Deprecate("7.0.0", "prospectors are deprecated, Use `inputs` instead.")
-			if len(tmpConfig.Filebeat.Inputs) > 0 {
-				return fmt.Errorf("prospectors and inputs used in the configuration file, define only inputs not both")
-			}
-			tmpConfig.Filebeat.Inputs = append(tmpConfig.Filebeat.Inputs, tmpConfig.Filebeat.Prospectors...)
-		}
-
 		config.Inputs = append(config.Inputs, tmpConfig.Filebeat.Inputs...)
 	}
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4669,19 +4669,6 @@ required: False
 Log stream when reading container logs, can be 'stdout' or 'stderr'
 
 
---
-
-*`prospector.type`*::
-+
---
-
-deprecated[6.3]
-
-required: True
-
-The input type from which the event was generated. This field is set to the value specified for the `type` option in the input section of the Filebeat config file. (DEPRECATED: see `input.type`)
-
-
 --
 
 *`input.type`*::

diff --git a/filebeat/docs/inputs/input-common-file-options.asciidoc b/filebeat/docs/inputs/input-common-file-options.asciidoc
@@ -1,5 +1,5 @@
 //////////////////////////////////////////////////////////////////////////
-//// This content is shared by Filebeat inputs that use the prospector
+//// This content is shared by Filebeat inputs that use the input
 //// to process files on disk (includes options for managing physical files)
 //// If you add IDs to sections, make sure you use attributes to create
 //// unique IDs for each input that includes this file. Use the format:
@@ -94,7 +94,7 @@ harvester is started and the latest changes will be picked up after
 We recommended that you set `close_inactive` to a value that is larger than the
 least frequent updates to your log files. For example, if your log files get
 updated every few seconds, you can safely set `close_inactive` to `1m`. If there
-are log files with very different update rates, you can use multiple 
+are log files with very different update rates, you can use multiple
 configurations with different values.
 
 Setting `close_inactive` to a lower value means that file handles are closed

diff --git a/filebeat/docs/inputs/input-common-harvester-options.asciidoc b/filebeat/docs/inputs/input-common-harvester-options.asciidoc
@@ -1,5 +1,5 @@
 //////////////////////////////////////////////////////////////////////////
-//// This content is shared by Filebeat inputs that use the prospector
+//// This content is shared by Filebeat inputs that use the input
 //// but do not process files (the options for managing files
 //// on disk are not relevant)
 //// If you add IDs to sections, make sure you use attributes to create

diff --git a/filebeat/fileset/config.go b/filebeat/fileset/config.go
@@ -35,26 +35,22 @@ type ModuleConfig struct {
 
 // FilesetConfig contains the configuration file options for a fileset
 type FilesetConfig struct {
-	Enabled    *bool                  `config:"enabled"`
-	Var        map[string]interface{} `config:"var"`
-	Input      map[string]interface{} `config:"input"`
-	Prospector map[string]interface{} `config:"prospector"`
+	Enabled *bool                  `config:"enabled"`
+	Var     map[string]interface{} `config:"var"`
+	Input   map[string]interface{} `config:"input"`
 }
 
 // NewFilesetConfig creates a new FilesetConfig from a common.Config.
 func NewFilesetConfig(cfg *common.Config) (*FilesetConfig, error) {
+	if err := cfgwarn.CheckRemoved6xSetting(cfg, "prospector"); err != nil {
+		return nil, err
+	}
+
 	var fcfg FilesetConfig
 	err := cfg.Unpack(&fcfg)
 	if err != nil {
 		return nil, fmt.Errorf("error unpacking configuration")
 	}
 
-	if len(fcfg.Prospector) > 0 {
-		cfgwarn.Deprecate("7.0.0", "prospector is deprecated. Use `input` instead.")
-		if len(fcfg.Input) > 0 {
-			return nil, fmt.Errorf("error prospector and input are defined in the fileset, use only input")
-		}
-		fcfg.Input = fcfg.Prospector
-	}
 	return &fcfg, nil
 }
diff --git a/filebeat/fileset/config_test.go b/filebeat/fileset/config_test.go
@@ -25,23 +25,6 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 )
 
-func TestProspectorDeprecation(t *testing.T) {
-	cfg := map[string]interface{}{
-		"enabled": true,
-		"prospector": map[string]interface{}{
-			"close_eof": true,
-		},
-	}
-
-	c, err := common.NewConfigFrom(cfg)
-	assert.NoError(t, err)
-
-	f, err := NewFilesetConfig(c)
-	if assert.NoError(t, err) {
-		assert.Equal(t, f.Input["close_eof"], true)
-	}
-}
-
 func TestInputSettings(t *testing.T) {
 	cfg := map[string]interface{}{
 		"enabled": true,
@@ -56,24 +39,5 @@ func TestInputSettings(t *testing.T) {
 	f, err := NewFilesetConfig(c)
 	if assert.NoError(t, err) {
 		assert.Equal(t, f.Input["close_eof"], true)
-		assert.Nil(t, f.Prospector)
 	}
 }
-
-func TestProspectorDeprecationWhenInputIsAlsoDefined(t *testing.T) {
-	cfg := map[string]interface{}{
-		"enabled": true,
-		"input": map[string]interface{}{
-			"close_eof": true,
-		},
-		"prospector": map[string]interface{}{
-			"close_eof": true,
-		},
-	}
-
-	c, err := common.NewConfigFrom(cfg)
-	assert.NoError(t, err)
-
-	_, err = NewFilesetConfig(c)
-	assert.Error(t, err, "error prospector and input are defined in the fileset, use only input")
-}
diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go
@@ -21,7 +21,7 @@ import (
 	"github.com/gofrs/uuid"
 
 	"github.com/elastic/beats/filebeat/channel"
-	input "github.com/elastic/beats/filebeat/prospector"
+	"github.com/elastic/beats/filebeat/input"
 	"github.com/elastic/beats/filebeat/registrar"
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/cfgfile"

diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go
@@ -34,6 +34,7 @@ import (
 	"text/template"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/logp"
 	mlimporter "github.com/elastic/beats/libbeat/ml-importer"
 )
@@ -102,7 +103,6 @@ type manifest struct {
 	Vars            []map[string]interface{} `config:"var"`
 	IngestPipeline  string                   `config:"ingest_pipeline"`
 	Input           string                   `config:"input"`
-	Prospector      string                   `config:"prospector"`
 	MachineLearning []struct {
 		Name       string `config:"name"`
 		Job        string `config:"job"`
@@ -115,14 +115,16 @@ type manifest struct {
 }
 
 func newManifest(cfg *common.Config) (*manifest, error) {
+	if err := cfgwarn.CheckRemoved6xSetting(cfg, "prospector"); err != nil {
+		return nil, err
+	}
+
 	var manifest manifest
 	err := cfg.Unpack(&manifest)
 	if err != nil {
 		return nil, err
 	}
-	if manifest.Prospector != "" {
-		manifest.Input = manifest.Prospector
-	}
+
 	return &manifest, nil
 }
 

diff --git a/filebeat/fileset/modules_test.go b/filebeat/fileset/modules_test.go
@@ -185,27 +185,6 @@ func TestApplyOverrides(t *testing.T) {
 				},
 			},
 		},
-		{
-			name:    "prospector overrides",
-			fcfg:    FilesetConfig{},
-			module:  "nginx",
-			fileset: "access",
-			overrides: &ModuleOverrides{
-				"nginx": map[string]*common.Config{
-					"access": load(t, map[string]interface{}{
-						"prospector.close_eof": true,
-					}),
-				},
-			},
-			expected: FilesetConfig{
-				Input: map[string]interface{}{
-					"close_eof": true,
-				},
-				Prospector: map[string]interface{}{
-					"close_eof": true,
-				},
-			},
-		},
 		{
 			name:    "input overrides",
 			fcfg:    FilesetConfig{},

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/input/docker/config.go b/filebeat/input/docker/config.go
@@ -29,7 +29,7 @@ var defaultConfig = config{
 type config struct {
 	Containers containers `config:"containers"`
 
-	// Partial configures the prospector to join partial lines
+	// Partial configures the input to join partial lines
 	Partial bool `config:"combine_partials"`
 
 	// Enable CRI flags parsing (to be switched to default in 7.0)

diff --git a/filebeat/input/log/config.go b/filebeat/input/log/config.go
@@ -145,7 +145,7 @@ var ValidScanSort = map[string]struct{}{
 }
 
 func (c *config) Validate() error {
-	// DEPRECATED 6.0.0: warning is already outputted on prospector level
+	// DEPRECATED 6.0.0: warning is already outputted on input level
 	if c.InputType != "" {
 		c.Type = c.InputType
 	}

diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json
@@ -11,8 +11,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
-        "offset": 0, 
-        "prospector.type": "log"
+        "offset": 0
     }, 
     {
         "@timestamp": "2016-12-26T16:22:13.000Z", 
@@ -36,8 +35,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
-        "offset": 73, 
-        "prospector.type": "log"
+        "offset": 73
     }, 
     {
         "@timestamp": "2016-12-26T14:16:48.000Z", 
@@ -47,8 +45,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
-        "offset": 238, 
-        "prospector.type": "log"
+        "offset": 238
     }, 
     {
         "@timestamp": "2017-05-29T19:02:48.000Z", 
@@ -71,7 +68,6 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
-        "offset": 285, 
-        "prospector.type": "log"
+        "offset": 285
     }
 ]
diff --git a/filebeat/module/apache2/error/test/test.log-expected.json b/filebeat/module/apache2/error/test/test.log-expected.json
@@ -7,8 +7,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
-        "offset": 0, 
-        "prospector.type": "log"
+        "offset": 0
     }, 
     {
         "@timestamp": "2016-12-26T16:15:55.103Z", 
@@ -19,8 +18,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
-        "offset": 99, 
-        "prospector.type": "log"
+        "offset": 99
     }, 
     {
         "@timestamp": "2011-09-09T10:42:29.902Z", 
@@ -33,7 +31,6 @@
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
-        "offset": 229, 
-        "prospector.type": "log"
+        "offset": 229
     }
 ]