Add --force-enable-module-filesets option to filebeat setup

Closes elastic#30916

CHANGELOG.next.asciidoc

@@ -235,6 +235,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Added support for Okta OAuth2 provider in the httpjson input. {pull}36273[36273]
 - Add support of the interval parameter in Salesforce setupaudittrail-rest fileset. {issue}35917[35917] {pull}35938[35938]
 - Add device handling to Okta input package for entity analytics. {pull}36049[36049]
+- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999]
 
 *Auditbeat*
 

filebeat/autodiscover/builder/hints/logs.go

@@ -69,7 +69,7 @@ func NewLogHints(cfg *conf.C) (autodiscover.Builder, error) {
 		return nil, fmt.Errorf("unable to unpack hints config due to error: %w", err)
 	}
 
-	moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, false)
+	moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, fileset.FilesetOverrides{})
 	if err != nil {
 		return nil, err
 	}

filebeat/beater/filebeat.go

@@ -108,7 +108,13 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
 		return nil, err
 	}
 
-	moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, false)
+	enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1)
+	forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1)
+	filesetOverrides := fileset.FilesetOverrides{
+		EnableAllFilesets:         enableAllFilesets,
+		ForceEnableModuleFilesets: forceEnableModuleFilesets,
+	}
+	moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, filesetOverrides)
 	if err != nil {
 		return nil, err
 	}
@@ -199,7 +205,13 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error {
 		// have to be loaded using cfg.Reloader. Otherwise those configurations are skipped.
 		pipelineLoaderFactory := newPipelineLoaderFactory(b.Config.Output.Config())
 		enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1)
-		modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, enableAllFilesets)
+		forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1)
+		filesetOverrides := fileset.FilesetOverrides{
+			EnableAllFilesets:         enableAllFilesets,
+			ForceEnableModuleFilesets: forceEnableModuleFilesets,
+		}
+
+		modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, filesetOverrides)
 		if fb.config.ConfigModules.Enabled() {
 			if enableAllFilesets {
 				// All module configs need to be loaded to enable all the filesets

filebeat/fileset/factory.go

@@ -129,7 +129,7 @@ func (f *Factory) CheckConfig(c *conf.C) error {
 // createRegistry starts a registry for a set of filesets, it returns the registry and
 // its input configurations
 func (f *Factory) createRegistry(c *conf.C) (*ModuleRegistry, []*conf.C, error) {
-	m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, false)
+	m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, FilesetOverrides{})
 	if err != nil {
 		return nil, nil, err
 	}

filebeat/fileset/modules.go

@@ -46,12 +46,17 @@ type Module struct {
 	config   ModuleConfig
 }
 
+type FilesetOverrides struct {
+	EnableAllFilesets         bool
+	ForceEnableModuleFilesets bool
+}
+
 // newModuleRegistry reads and loads the configured module into the registry.
 func newModuleRegistry(modulesPath string,
 	moduleConfigs []*ModuleConfig,
 	overrides *ModuleOverrides,
 	beatInfo beat.Info,
-	enableAllFilesets bool,
+	filesetOverrides FilesetOverrides,
 ) (*ModuleRegistry, error) {
 	reg := ModuleRegistry{
 		registry: []Module{},
@@ -61,7 +66,7 @@ func newModuleRegistry(modulesPath string,
 	for _, mcfg := range moduleConfigs {
 		// an empty ModuleConfig can reach this so we only force enable a
 		// config if the Module name is set and Enabled pointer is valid.
-		if enableAllFilesets && mcfg.Module != "" && mcfg.Enabled != nil {
+		if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && mcfg.Module != "" && mcfg.Enabled != nil {
 			*mcfg.Enabled = true
 		}
 		if mcfg.Module == "" || (mcfg.Enabled != nil && !(*mcfg.Enabled)) {
@@ -80,16 +85,26 @@ func newModuleRegistry(modulesPath string,
 			config:   *mcfg,
 			filesets: []Fileset{},
 		}
-		for filesetName, fcfg := range mcfg.Filesets {
+		if filesetOverrides.ForceEnableModuleFilesets {
+			if mcfg.Filesets == nil {
+				mcfg.Filesets = make(map[string]*FilesetConfig)
+			}
+			for _, fName := range moduleFilesets {
+				if _, ok := mcfg.Filesets[fName]; !ok {
+					mcfg.Filesets[fName] = &FilesetConfig{Enabled: func() *bool { b := true; return &b }()}
+				}
+			}
+		}
 
+		for filesetName, fcfg := range mcfg.Filesets {
 			fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides)
 			if err != nil {
 				return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %w", mcfg.Module, filesetName, err)
 			}
 
 			// ModuleConfig can have empty Filesets so we only force
 			// enable if the Enabled pointer is valid
-			if enableAllFilesets && fcfg.Enabled != nil {
+			if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && fcfg.Enabled != nil {
 				*fcfg.Enabled = true
 			}
 			if fcfg.Enabled != nil && !(*fcfg.Enabled) {
@@ -128,7 +143,7 @@ func newModuleRegistry(modulesPath string,
 }
 
 // NewModuleRegistry reads and loads the configured module into the registry.
-func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, enableAllFilesets bool) (*ModuleRegistry, error) {
+func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, filesetOverrides FilesetOverrides) (*ModuleRegistry, error) {
 	modulesPath := paths.Resolve(paths.Home, "module")
 
 	stat, err := os.Stat(modulesPath)
@@ -166,7 +181,7 @@ func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, e
 	}
 
 	enableFilesetsFromOverrides(mcfgs, modulesOverrides)
-	return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, enableAllFilesets)
+	return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, filesetOverrides)
 }
 
 // enableFilesetsFromOverrides enables in mcfgs the filesets mentioned in overrides,

filebeat/fileset/modules_integration_test.go

@@ -114,7 +114,7 @@ func TestSetupNginx(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), FilesetOverrides{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -193,7 +193,7 @@ func TestLoadMultiplePipelines(t *testing.T) {
 		{"foo", &enabled, filesetConfigs},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -238,7 +238,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) {
 		{"foo", &enabled, filesetConfigs},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{})
 	if err != nil {
 		t.Fatal(err)
 	}

filebeat/fileset/modules_test.go

@@ -81,7 +81,7 @@ func TestNewModuleRegistry(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 
@@ -148,7 +148,7 @@ func TestNewModuleRegistryConfig(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 
@@ -174,7 +174,7 @@ func TestMovedModule(t *testing.T) {
 		},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false)
+	reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 }
@@ -445,7 +445,7 @@ func TestMissingModuleFolder(t *testing.T) {
 		load(t, map[string]interface{}{"module": "nginx"}),
 	}
 
-	reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, false)
+	reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, FilesetOverrides{})
 	require.NoError(t, err)
 	assert.NotNil(t, reg)
 

filebeat/fileset/setup.go

@@ -29,22 +29,22 @@ type SetupFactory struct {
 	beatInfo              beat.Info
 	pipelineLoaderFactory PipelineLoaderFactory
 	overwritePipelines    bool
-	enableAllFilesets     bool
+	filesetOverrides      FilesetOverrides
 }
 
 // NewSetupFactory creates a SetupFactory
-func NewSetupFactory(beatInfo beat.Info, pipelineLoaderFactory PipelineLoaderFactory, enableAllFilesets bool) *SetupFactory {
+func NewSetupFactory(beatInfo beat.Info, pipelineLoaderFactory PipelineLoaderFactory, filesetOverrides FilesetOverrides) *SetupFactory {
 	return &SetupFactory{
 		beatInfo:              beatInfo,
 		pipelineLoaderFactory: pipelineLoaderFactory,
 		overwritePipelines:    true,
-		enableAllFilesets:     enableAllFilesets,
+		filesetOverrides:      filesetOverrides,
 	}
 }
 
 // Create creates a new SetupCfgRunner to setup module configuration.
 func (sf *SetupFactory) Create(_ beat.PipelineConnector, c *conf.C) (cfgfile.Runner, error) {
-	m, err := NewModuleRegistry([]*conf.C{c}, sf.beatInfo, false, sf.enableAllFilesets)
+	m, err := NewModuleRegistry([]*conf.C{c}, sf.beatInfo, false, sf.filesetOverrides)
 	if err != nil {
 		return nil, err
 	}