From 158c0c093e71f9728010eb1a5f35cca0602c5ae2 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 31 Jul 2023 08:21:07 -0500 Subject: [PATCH] Add --force-enable-module-filesets option to filebeat setup Closes #30916 --- CHANGELOG.next.asciidoc | 1 + filebeat/autodiscover/builder/hints/logs.go | 2 +- filebeat/beater/filebeat.go | 16 +- filebeat/fileset/factory.go | 2 +- filebeat/fileset/modules.go | 27 ++- filebeat/fileset/modules_integration_test.go | 6 +- filebeat/fileset/modules_test.go | 8 +- filebeat/fileset/setup.go | 8 +- .../tests/integration/setup_pipelines_test.go | 200 ++++++++++++++++++ go.mod | 1 + go.sum | 3 + libbeat/cmd/instance/beat.go | 20 +- libbeat/cmd/setup.go | 36 ++-- 13 files changed, 287 insertions(+), 43 deletions(-) create mode 100644 filebeat/tests/integration/setup_pipelines_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 75712ec28982..355f0e376cd4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -235,6 +235,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Added support for Okta OAuth2 provider in the httpjson input. {pull}36273[36273] - Add support of the interval parameter in Salesforce setupaudittrail-rest fileset. {issue}35917[35917] {pull}35938[35938] - Add device handling to Okta input package for entity analytics. {pull}36049[36049] +- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999] *Auditbeat* diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 59d2eea70ae1..85411a34bba5 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -69,7 +69,7 @@ func NewLogHints(cfg *conf.C) (autodiscover.Builder, error) { return nil, fmt.Errorf("unable to unpack hints config due to error: %w", err) } - moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, false) + moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false, fileset.FilesetOverrides{}) if err != nil { return nil, err } diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 532d8132110f..2fddfd3bb4fb 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -108,7 +108,13 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea return nil, err } - moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, false) + enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1) + forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1) + filesetOverrides := fileset.FilesetOverrides{ + EnableAllFilesets: enableAllFilesets, + ForceEnableModuleFilesets: forceEnableModuleFilesets, + } + moduleRegistry, err := fileset.NewModuleRegistry(config.Modules, b.Info, true, filesetOverrides) if err != nil { return nil, err } @@ -199,7 +205,13 @@ func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error { // have to be loaded using cfg.Reloader. Otherwise those configurations are skipped. pipelineLoaderFactory := newPipelineLoaderFactory(b.Config.Output.Config()) enableAllFilesets, _ := b.BeatConfig.Bool("config.modules.enable_all_filesets", -1) - modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, enableAllFilesets) + forceEnableModuleFilesets, _ := b.BeatConfig.Bool("config.modules.force_enable_module_filesets", -1) + filesetOverrides := fileset.FilesetOverrides{ + EnableAllFilesets: enableAllFilesets, + ForceEnableModuleFilesets: forceEnableModuleFilesets, + } + + modulesFactory := fileset.NewSetupFactory(b.Info, pipelineLoaderFactory, filesetOverrides) if fb.config.ConfigModules.Enabled() { if enableAllFilesets { // All module configs need to be loaded to enable all the filesets diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index bdc6350618e2..0e5308946fa2 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -129,7 +129,7 @@ func (f *Factory) CheckConfig(c *conf.C) error { // createRegistry starts a registry for a set of filesets, it returns the registry and // its input configurations func (f *Factory) createRegistry(c *conf.C) (*ModuleRegistry, []*conf.C, error) { - m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, false) + m, err := NewModuleRegistry([]*conf.C{c}, f.beatInfo, false, FilesetOverrides{}) if err != nil { return nil, nil, err } diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index 158b45f56d39..572b929dcb2c 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -46,12 +46,17 @@ type Module struct { config ModuleConfig } +type FilesetOverrides struct { + EnableAllFilesets bool + ForceEnableModuleFilesets bool +} + // newModuleRegistry reads and loads the configured module into the registry. func newModuleRegistry(modulesPath string, moduleConfigs []*ModuleConfig, overrides *ModuleOverrides, beatInfo beat.Info, - enableAllFilesets bool, + filesetOverrides FilesetOverrides, ) (*ModuleRegistry, error) { reg := ModuleRegistry{ registry: []Module{}, @@ -61,7 +66,7 @@ func newModuleRegistry(modulesPath string, for _, mcfg := range moduleConfigs { // an empty ModuleConfig can reach this so we only force enable a // config if the Module name is set and Enabled pointer is valid. - if enableAllFilesets && mcfg.Module != "" && mcfg.Enabled != nil { + if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && mcfg.Module != "" && mcfg.Enabled != nil { *mcfg.Enabled = true } if mcfg.Module == "" || (mcfg.Enabled != nil && !(*mcfg.Enabled)) { @@ -80,8 +85,18 @@ func newModuleRegistry(modulesPath string, config: *mcfg, filesets: []Fileset{}, } - for filesetName, fcfg := range mcfg.Filesets { + if filesetOverrides.ForceEnableModuleFilesets { + if mcfg.Filesets == nil { + mcfg.Filesets = make(map[string]*FilesetConfig) + } + for _, fName := range moduleFilesets { + if _, ok := mcfg.Filesets[fName]; !ok { + mcfg.Filesets[fName] = &FilesetConfig{Enabled: func() *bool { b := true; return &b }()} + } + } + } + for filesetName, fcfg := range mcfg.Filesets { fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides) if err != nil { return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %w", mcfg.Module, filesetName, err) @@ -89,7 +104,7 @@ func newModuleRegistry(modulesPath string, // ModuleConfig can have empty Filesets so we only force // enable if the Enabled pointer is valid - if enableAllFilesets && fcfg.Enabled != nil { + if (filesetOverrides.EnableAllFilesets || filesetOverrides.ForceEnableModuleFilesets) && fcfg.Enabled != nil { *fcfg.Enabled = true } if fcfg.Enabled != nil && !(*fcfg.Enabled) { @@ -128,7 +143,7 @@ func newModuleRegistry(modulesPath string, } // NewModuleRegistry reads and loads the configured module into the registry. -func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, enableAllFilesets bool) (*ModuleRegistry, error) { +func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, filesetOverrides FilesetOverrides) (*ModuleRegistry, error) { modulesPath := paths.Resolve(paths.Home, "module") stat, err := os.Stat(modulesPath) @@ -166,7 +181,7 @@ func NewModuleRegistry(moduleConfigs []*conf.C, beatInfo beat.Info, init bool, e } enableFilesetsFromOverrides(mcfgs, modulesOverrides) - return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, enableAllFilesets) + return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo, filesetOverrides) } // enableFilesetsFromOverrides enables in mcfgs the filesets mentioned in overrides, diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go index 2d06db83c093..bb8b7c6dd882 100644 --- a/filebeat/fileset/modules_integration_test.go +++ b/filebeat/fileset/modules_integration_test.go @@ -114,7 +114,7 @@ func TestSetupNginx(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), false) + reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"), FilesetOverrides{}) if err != nil { t.Fatal(err) } @@ -193,7 +193,7 @@ func TestLoadMultiplePipelines(t *testing.T) { {"foo", &enabled, filesetConfigs}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false) + reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{}) if err != nil { t.Fatal(err) } @@ -238,7 +238,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) { {"foo", &enabled, filesetConfigs}, } - reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), false) + reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"), FilesetOverrides{}) if err != nil { t.Fatal(err) } diff --git a/filebeat/fileset/modules_test.go b/filebeat/fileset/modules_test.go index f1baed9e5c50..2328e327c004 100644 --- a/filebeat/fileset/modules_test.go +++ b/filebeat/fileset/modules_test.go @@ -81,7 +81,7 @@ func TestNewModuleRegistry(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false) + reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) @@ -148,7 +148,7 @@ func TestNewModuleRegistryConfig(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false) + reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) @@ -174,7 +174,7 @@ func TestMovedModule(t *testing.T) { }, } - reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, false) + reg, err := newModuleRegistry(modulesPath, configs, nil, beat.Info{Version: "5.2.0"}, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) } @@ -445,7 +445,7 @@ func TestMissingModuleFolder(t *testing.T) { load(t, map[string]interface{}{"module": "nginx"}), } - reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, false) + reg, err := NewModuleRegistry(configs, beat.Info{Version: "5.2.0"}, true, FilesetOverrides{}) require.NoError(t, err) assert.NotNil(t, reg) diff --git a/filebeat/fileset/setup.go b/filebeat/fileset/setup.go index 3ef5a6f2997e..a17a504ab7f3 100644 --- a/filebeat/fileset/setup.go +++ b/filebeat/fileset/setup.go @@ -29,22 +29,22 @@ type SetupFactory struct { beatInfo beat.Info pipelineLoaderFactory PipelineLoaderFactory overwritePipelines bool - enableAllFilesets bool + filesetOverrides FilesetOverrides } // NewSetupFactory creates a SetupFactory -func NewSetupFactory(beatInfo beat.Info, pipelineLoaderFactory PipelineLoaderFactory, enableAllFilesets bool) *SetupFactory { +func NewSetupFactory(beatInfo beat.Info, pipelineLoaderFactory PipelineLoaderFactory, filesetOverrides FilesetOverrides) *SetupFactory { return &SetupFactory{ beatInfo: beatInfo, pipelineLoaderFactory: pipelineLoaderFactory, overwritePipelines: true, - enableAllFilesets: enableAllFilesets, + filesetOverrides: filesetOverrides, } } // Create creates a new SetupCfgRunner to setup module configuration. func (sf *SetupFactory) Create(_ beat.PipelineConnector, c *conf.C) (cfgfile.Runner, error) { - m, err := NewModuleRegistry([]*conf.C{c}, sf.beatInfo, false, sf.enableAllFilesets) + m, err := NewModuleRegistry([]*conf.C{c}, sf.beatInfo, false, sf.filesetOverrides) if err != nil { return nil, err } diff --git a/filebeat/tests/integration/setup_pipelines_test.go b/filebeat/tests/integration/setup_pipelines_test.go new file mode 100644 index 000000000000..934d9dc4dfb7 --- /dev/null +++ b/filebeat/tests/integration/setup_pipelines_test.go @@ -0,0 +1,200 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package integration + +import ( + "fmt" + "net/http" + "net/url" + "path/filepath" + "testing" + "time" + + cp "github.com/otiai10/copy" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/tests/integration" +) + +func TestSetupNoModules(t *testing.T) { + cfg := ` +filebeat: +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module direcotry") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + filebeat.Start("setup") + filebeat.WaitForLogs("Setup called, but no modules enabled.", 10*time.Second) +} + +func TestSetupModulesNoFileset(t *testing.T) { + cfg := ` +filebeat.config: + modules: + enabled: true + path: modules.d/*.yml +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +logging.level: debug +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module direcotry") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + filebeat.Start("setup", "--pipelines") + filebeat.WaitForLogs("Number of module configs found: 0", 10*time.Second) +} + +func TestSetupModulesOneEnabled(t *testing.T) { + cfg := ` +filebeat.config: + modules: + enabled: true + path: modules.d/*.yml +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +logging.level: debug +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module direcotry") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + filebeat.Start("setup", "--pipelines", "--modules", "apache") + filebeat.WaitForLogs("Exiting: module apache is configured but has no enabled filesets", 10*time.Second) +} + +func TestSetupModulesOneEnabledOverride(t *testing.T) { + cfg := ` +filebeat.config: + modules: + enabled: true + path: modules.d/*.yml +output.elasticsearch: + hosts: + - %s + username: %s + password: %s + allow_older_versions: true +setup.kibana: + hosts: %s + username: %s + password: %s +logging.level: debug +` + integration.EnsureESIsRunning(t) + esURL := integration.GetESURL(t, "http") + esPassword, ok := esURL.User.Password() + require.True(t, ok, "ES didn't have a password") + kURL, kUserInfo := integration.GetKibana(t) + kPassword, ok := kUserInfo.Password() + require.True(t, ok, "Kibana didn't have a password") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + + err := cp.Copy("../../module", filepath.Join(filebeat.TempDir(), "module")) + require.NoError(t, err, "error copying module direcotry") + + err = cp.Copy("../../modules.d", filepath.Join(filebeat.TempDir(), "modules.d")) + require.NoError(t, err, "error copying modules.d directory") + + filebeat.WriteConfigFile(fmt.Sprintf(cfg, esURL.Host, esURL.User.Username(), esPassword, kURL.Host, kUserInfo.Username(), kPassword)) + + t.Cleanup(func() { + delURL, err := url.Parse(esURL.String()) + if err != nil { + t.Fatalf("could not parse ES url: %s", err) + } + delURL.Path = "/_ingest/pipeline/filebeat-*" + ret, _, err := integration.HttpDo(t, http.MethodDelete, *delURL) + if err != nil { + t.Logf("error while deleting filebeat-* pipelines: %s", err) + } + if ret != http.StatusOK { + t.Logf("status was %d while deleting filebeat-* pipelines", ret) + } + }) + filebeat.Start("setup", "--pipelines", "--modules", "apache", "--force-enable-module-filesets") + filebeat.WaitForLogs("Elasticsearch pipeline loaded.", 10*time.Second) + filebeat.WaitForLogs("Elasticsearch pipeline loaded.", 10*time.Second) +} diff --git a/go.mod b/go.mod index 67c9c980f5b9..6df03098b112 100644 --- a/go.mod +++ b/go.mod @@ -331,6 +331,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/otiai10/copy v1.12.0 // indirect github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect diff --git a/go.sum b/go.sum index 4cca126049fa..136a1c7c8398 100644 --- a/go.sum +++ b/go.sum @@ -786,6 +786,7 @@ github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= @@ -1325,6 +1326,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/osquery/osquery-go v0.0.0-20220706183148-4e1f83012b42 h1:Epwxipb+y/e8ss/SJ7947F8J6dwjv3RHRCz2g0OkCII= github.com/osquery/osquery-go v0.0.0-20220706183148-4e1f83012b42/go.mod h1:0KzmMhe0PL19cdYq6nd1cT9/5bMMJBTssAfuEgM2i34= +github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= +github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 0f1d44a777c6..b881fc4bdeec 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -201,7 +201,6 @@ func initRand() { // instance. // XXX Move this as a *Beat method? func Run(settings Settings, bt beat.Creator) error { - return handleError(func() error { defer func() { if r := recover(); r != nil { @@ -502,7 +501,7 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { } ctx, cancel := context.WithCancel(context.Background()) - var stopBeat = func() { + stopBeat := func() { b.Instrumentation.Tracer().Close() beater.Stop() } @@ -615,8 +614,9 @@ type SetupSettings struct { // Deprecated: use IndexManagementKey instead Template bool // Deprecated: use IndexManagementKey instead - ILMPolicy bool - EnableAllFilesets bool + ILMPolicy bool + EnableAllFilesets bool + ForceEnableModuleFilesets bool } // Setup registers ES index template, kibana dashboards, ml jobs and pipelines. @@ -628,16 +628,19 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if err != nil { return err } - // Tell the beat that we're in the setup command b.InSetupCmd = true + if setup.ForceEnableModuleFilesets { + if err := b.Beat.BeatConfig.SetBool("config.modules.force_enable_module_filesets", -1, true); err != nil { + return fmt.Errorf("error setting force_enable_module_filesets config option %w", err) + } + } // Create beater to give it the opportunity to set loading callbacks _, err = b.createBeater(bt) if err != nil { return err } - if setup.IndexManagement || setup.Template || setup.ILMPolicy { outCfg := b.Config.Output if !isElasticsearchOutput(outCfg.Name()) { @@ -648,7 +651,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return err } - var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset + loadTemplate, loadILM := idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset if setup.IndexManagement || setup.Template { loadTemplate = idxmgmt.LoadModeOverwrite } @@ -687,6 +690,7 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return fmt.Errorf("error setting enable_all_filesets config option %w", err) } } + esConfig := b.Config.Output.Config() err = b.OverwritePipelinesCallback(esConfig) if err != nil { @@ -900,7 +904,7 @@ func (b *Beat) loadMeta(metaPath string) error { // write temporary file first tempFile := metaPath + ".new" - f, err = os.OpenFile(tempFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + f, err = os.OpenFile(tempFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return fmt.Errorf("failed to create Beat meta file: %w", err) } diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index f1d2432ead94..49a2cb5cfbea 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -28,14 +28,18 @@ import ( ) const ( - //DashboardKey used for registering dashboards in setup cmd + // DashboardKey used for registering dashboards in setup cmd DashboardKey = "dashboards" - //PipelineKey used for registering pipelines in setup cmd + // PipelineKey used for registering pipelines in setup cmd PipelineKey = "pipelines" - //IndexManagementKey used for loading all components related to ES index management in setup cmd + // IndexManagementKey used for loading all components related to ES index management in setup cmd IndexManagementKey = "index-management" - //EnableAllFilesetsKey enables all modules and filesets regardless of config + // EnableAllFilesetsKey enables all modules and filesets regardless of config EnableAllFilesetsKey = "enable-all-filesets" + // ForceEnableModuleFilesets enables all the filesets contained + // in the modules that have been explicitly enabled. The + // requires "modules" to be used. + ForceEnableModuleFilesets = "force-enable-module-filesets" ) func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Command { @@ -56,32 +60,33 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co os.Exit(1) } - var registeredFlags = map[string]bool{ - DashboardKey: false, - PipelineKey: false, - IndexManagementKey: false, - EnableAllFilesetsKey: false, + registeredFlags := map[string]bool{ + DashboardKey: false, + PipelineKey: false, + IndexManagementKey: false, + EnableAllFilesetsKey: false, + ForceEnableModuleFilesets: false, } - var setupAll = true + setupAll := true // create collection with registered flags and their values for k := range registeredFlags { val, err := cmd.Flags().GetBool(k) - //if flag is not registered, an error is thrown + // if flag is not registered, an error is thrown if err != nil { delete(registeredFlags, k) continue } registeredFlags[k] = val - //if any flag is set via cmd line then only this flag should be run + // if any flag is set via cmd line then only this flag should be run if val { setupAll = false } } - //create the struct to pass on - var s = instance.SetupSettings{} + // create the struct to pass on + s := instance.SetupSettings{} for k, v := range registeredFlags { if setupAll || v { switch k { @@ -93,6 +98,8 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co s.IndexManagement = true case EnableAllFilesetsKey: s.EnableAllFilesets = true + case ForceEnableModuleFilesets: + s.ForceEnableModuleFilesets = true } } } @@ -108,6 +115,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co setup.Flags().Bool(IndexManagementKey, false, "Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias") setup.Flags().Bool("enable-all-filesets", false, "Behave as if all modules and filesets had been enabled") + setup.Flags().Bool("force-enable-module-filesets", false, "Behave as if all filesets, within enabled modules, are enabled") return &setup }