diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2cb6955df74d..b51ef5b8e86f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -309,6 +309,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update cloud.google.com/go library. {pull}28229[28229] - Update ECS to 1.12.0. {pull}27770[27770] - Fields mapped as `match_only_text` will automatically fallback to a `text` mapping when using Elasticsearch versions that do not support `match_only_text`. {pull}27770[27770] +- Do not load ML jobs to Elasticsearch 8.x from new Beats 7.x releases. {pull}27771[27771] *Auditbeat* diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index 4cd5d233f6b1..008bb36860b8 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -44,6 +44,7 @@ import ( "github.com/elastic/beats/v7/libbeat/kibana" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/management" + mlimporter "github.com/elastic/beats/v7/libbeat/ml-importer" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch" "github.com/elastic/beats/v7/libbeat/publisher/pipetool" @@ -160,8 +161,8 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *common.Config) (b } // register `setup` callback for ML jobs - b.SetupMLCallback = func(b *beat.Beat, kibanaConfig *common.Config) error { - return fb.loadModulesML(b, kibanaConfig) + b.SetupMLCallback = func(b *beat.Beat, fromFlag bool, kibanaConfig *common.Config) error { + return fb.loadModulesML(b, fromFlag, kibanaConfig) } err = fb.setupPipelineLoaderCallback(b) @@ -223,7 +224,7 @@ func (fb *Filebeat) loadModulesPipelines(b *beat.Beat) error { return err } -func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) error { +func (fb *Filebeat) loadModulesML(b *beat.Beat, fromFlag bool, kibanaConfig *common.Config) error { var errs multierror.Errors logp.Debug("machine-learning", "Setting up ML jobs for modules") @@ -261,7 +262,7 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err return errors.Errorf("Error creating Kibana client: %v", err) } - if err := setupMLBasedOnVersion(fb.moduleRegistry, esClient, kibanaClient); err != nil { + if err := setupMLBasedOnVersion(fb.moduleRegistry, fromFlag, esClient, kibanaClient); err != nil { errs = append(errs, err) } @@ -287,17 +288,23 @@ func (fb *Filebeat) loadModulesML(b *beat.Beat, kibanaConfig *common.Config) err continue } - if err := setupMLBasedOnVersion(set, esClient, kibanaClient); err != nil { + if err := setupMLBasedOnVersion(set, fromFlag, esClient, kibanaClient); err != nil { errs = append(errs, err) } } } + if len(errs) == 0 { + fmt.Println("Loaded machine learning job configurations") + } return errs.Err() } -func setupMLBasedOnVersion(reg *fileset.ModuleRegistry, esClient *eslegclient.Connection, kibanaClient *kibana.Client) error { +func setupMLBasedOnVersion(reg *fileset.ModuleRegistry, fromFlag bool, esClient *eslegclient.Connection, kibanaClient *kibana.Client) error { + if !mlimporter.IsCompatible(esClient) && fromFlag { + return fmt.Errorf("Machine learning jobs are not loaded because Elasticsearch version is too new. It must be 7.x for setting up ML using Beats. Use the Machine learning UI in Kibana.") + } if isElasticsearchLoads(kibanaClient.GetVersion()) { return reg.LoadML(esClient) } diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index b748bc3858a4..69b49b013b6e 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -392,6 +392,11 @@ func checkAvailableProcessors(esClient PipelineLoader, requiredProcessors []Proc // LoadML loads the machine-learning configurations into Elasticsearch, if X-Pack is available func (reg *ModuleRegistry) LoadML(esClient PipelineLoader) error { + if !mlimporter.IsCompatible(esClient) { + logp.Info("Skipping loading machine learning jobs because of Elasticsearch version is too new.\nIt must be 7.x for setting up ML using Beats. Please use the Machine Learning UI in Kibana.") + return nil + } + haveXpack, err := mlimporter.HaveXpackML(esClient) if err != nil { return errors.Errorf("error checking if xpack is available: %v", err) @@ -417,6 +422,11 @@ func (reg *ModuleRegistry) LoadML(esClient PipelineLoader) error { // SetupML sets up the machine-learning configurations into Elasticsearch using Kibana, if X-Pack is available func (reg *ModuleRegistry) SetupML(esClient PipelineLoader, kibanaClient *kibana.Client) error { + if !mlimporter.IsCompatible(esClient) { + logp.Info("Skipping loading machine learning jobs because of Elasticsearch version is too new.\nIt must be 7.x for setting up it using Beats. Please use the Machine Learning UI in Kibana.") + return nil + } + haveXpack, err := mlimporter.HaveXpackML(esClient) if err != nil { return errors.Errorf("Error checking if xpack is available: %v", err) diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 9413fcae087f..3e5e8f325035 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -90,7 +90,7 @@ type BeatConfig struct { // SetupMLCallback can be used by the Beat to register MachineLearning configurations // for the enabled modules. -type SetupMLCallback func(*Beat, *common.Config) error +type SetupMLCallback func(*Beat, bool, *common.Config) error // OverwritePipelinesCallback can be used by the Beat to register Ingest pipeline loader // for the enabled modules. diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 83f213c0fd4f..780a8f428d30 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -508,6 +508,7 @@ type SetupSettings struct { Template bool //Deprecated: use IndexManagementKey instead ILMPolicy bool + SetupAll bool } // Setup registers ES index template, kibana dashboards, ml jobs and pipelines. @@ -573,11 +574,12 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er if setup.MachineLearning && b.SetupMLCallback != nil { cfgwarn.Deprecate("8.0.0", "Setting up ML using %v is going to be removed. Please use the ML app to setup jobs.", strings.Title(b.Info.Beat)) fmt.Println("Setting up ML using setup --machine-learning is going to be removed in 8.0.0. Please use the ML app instead.\nSee more: https://www.elastic.co/guide/en/machine-learning/current/index.html") - err = b.SetupMLCallback(&b.Beat, b.Config.Kibana) + fmt.Println("It is not possble to load ML jobs into an Elasticsearch 8.0.0 or newer using the Beat.") + + err = b.SetupMLCallback(&b.Beat, !setup.SetupAll, b.Config.Kibana) if err != nil { return err } - fmt.Println("Loaded machine learning job configurations") } if setup.Pipeline && b.OverwritePipelinesCallback != nil { diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index 02e16ca3299f..17a37700eeaa 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -94,7 +94,7 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co } //create the struct to pass on - var s = instance.SetupSettings{} + var s = instance.SetupSettings{SetupAll: setupAll} for k, v := range registeredFlags { if setupAll || v { switch k { diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index bb8f8022abec..92ce103769f5 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -752,7 +752,9 @@ endif::no_dashboards[] ifdef::has_ml_jobs[] * The machine learning jobs contain the configuration information and metadata -necessary to analyze data for anomalies. +necessary to analyze data for anomalies. You can use this flag to load machine learning jobs +into Elasticsearch version 7. For version 8 and later, use the Machine learning UI in Kibana +as described in the {ml-docs}/create-jobs.html[documentation]. endif::[] This command sets up the environment without actually running @@ -783,6 +785,7 @@ Shows help for the `setup` command. ifdef::has_ml_jobs[] *`--machine-learning`*:: +deprecated:[7.12] Sets up machine learning job configurations only. endif::[] diff --git a/libbeat/ml-importer/importer.go b/libbeat/ml-importer/importer.go index 9d35382e9fda..db60f083120d 100644 --- a/libbeat/ml-importer/importer.go +++ b/libbeat/ml-importer/importer.go @@ -40,6 +40,9 @@ var ( kibanaGetModuleURL = "/api/ml/modules/get_module/%s" kibanaRecognizeURL = "/api/ml/modules/recognize/%s" kibanaSetupModuleURL = "/api/ml/modules/setup/%s" + + // ML assets are not loaded to ES 8.0.0 or newer. + incompatibleSince = common.MustNewVersion("8.0.0") ) // MLConfig contains the required configuration for loading one job and the associated @@ -177,6 +180,13 @@ func ImportMachineLearningJob(esClient MLLoader, cfg *MLConfig) error { return nil } +// IsCompatible checks if the version of Elasticsearch is supported. +// Beats does not load ML assets to 8.0 or newer. +func IsCompatible(esClient MLLoader) bool { + esVersion := esClient.GetVersion() + return esVersion.LessThan(incompatibleSince) +} + // HaveXpackML checks whether X-pack is installed and has Machine Learning enabled. func HaveXpackML(esClient MLLoader) (bool, error) { status, response, err := esClient.Request("GET", "/_xpack", "", nil, nil)