diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 34031ebe0dc..0ad71e05637 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -25,6 +25,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671] - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620] - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620] +- Use data streams instead of indices for storing events from Beats. {pull}28450[28450] +- Remove option `setup.template.type` and always load composable template with data streams. {pull}28450[28450] +- Remove several ILM options (`rollover_alias` and `pattern`) as data streams does not require index aliases. {pull}28450[28450] - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215] - Remove deprecated `--template` and `--ilm-policy` flags. Use `--index-management` instead. {pull}28870[28870] - Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927] diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 24bfac14414..fc685b6c231 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1145,19 +1145,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default auditbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "auditbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "auditbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "auditbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "auditbeat-%{[agent.version]}-*" +#setup.template.pattern: "auditbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1204,17 +1198,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'auditbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'auditbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 9b6a421d6c4..a8217924d4d 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2078,19 +2078,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default filebeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "filebeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "filebeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "filebeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "filebeat-%{[agent.version]}-*" +#setup.template.pattern: "filebeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -2137,17 +2131,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'filebeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'filebeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 4f74323ead1..f3b250c6b49 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -120,14 +120,8 @@ filebeat.config.{{ reload_type|default("inputs") }}: {% if ilm %} setup.ilm: - enabled: {{ ilm.enabled | default("auto") }} + enabled: {{ ilm.enabled | default("true") }} policy_name: libbeat-test-default-policy - {% if ilm.pattern %} - pattern: {{ ilm.pattern }} - {% endif %} - {% if ilm.rollover_alias %} - rollover_alias: {{ ilm.rollover_alias }} - {% endif %} {% endif %} diff --git a/filebeat/tests/system/config/filebeat_inputs.yml.j2 b/filebeat/tests/system/config/filebeat_inputs.yml.j2 index f9a6d725a92..9757f60023a 100644 --- a/filebeat/tests/system/config/filebeat_inputs.yml.j2 +++ b/filebeat/tests/system/config/filebeat_inputs.yml.j2 @@ -17,14 +17,8 @@ filebeat.registry: {% if ilm %} setup.ilm: - enabled: {{ ilm.enabled | default("auto") }} + enabled: {{ ilm.enabled | default("true") }} policy_name: libbeat-test-default-policy - {% if ilm.pattern %} - pattern: {{ ilm.pattern }} - {% endif %} - {% if ilm.rollover_alias %} - rollover_alias: {{ ilm.rollover_alias }} - {% endif %} {% endif %} diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index b552560f83c..b4ee0c7b487 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -104,7 +104,7 @@ def run_on_file(self, module, fileset, test_file, cfgfile): self.assert_explicit_ecs_version_set(module, fileset) try: - self.es.indices.delete(index=self.index_name) + self.es.indices.delete_data_stream(self.index_name) except BaseException: pass self.wait_until(lambda: not self.es.indices.exists(self.index_name)) @@ -242,7 +242,7 @@ def clean_keys(obj): host_keys.append("host.name") # The create timestamps area always new - time_keys = ["event.created", "event.ingested"] + time_keys = ["event.created", "event.ingested", "@timestamp"] # source path and agent.version can be different for each run other_keys = ["log.file.path", "agent.version"] # ECS versions change for any ECS release, large or small diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 8f0f019626b..c3ae5c01214 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1291,19 +1291,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default heartbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "heartbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "heartbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "heartbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "heartbeat-%{[agent.version]}-*" +#setup.template.pattern: "heartbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1350,17 +1344,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'heartbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'heartbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl index 1416da0cc8d..5e4a2e642fe 100644 --- a/libbeat/_meta/config/setup.ilm.reference.yml.tmpl +++ b/libbeat/_meta/config/setup.ilm.reference.yml.tmpl @@ -5,17 +5,8 @@ # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is '{{.BeatName}}-%{[agent.version]}'. -#setup.ilm.rollover_alias: '{{.BeatIndexPrefix}}' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/libbeat/_meta/config/setup.template.reference.yml.tmpl b/libbeat/_meta/config/setup.template.reference.yml.tmpl index 95d8d4a50c8..d3019c6f2b8 100644 --- a/libbeat/_meta/config/setup.template.reference.yml.tmpl +++ b/libbeat/_meta/config/setup.template.reference.yml.tmpl @@ -7,19 +7,13 @@ # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default {{.BeatName}} uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "{{.BeatIndexPrefix}}-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "{{.BeatIndexPrefix}}-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "{{.BeatIndexPrefix}}-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "{{.BeatIndexPrefix}}-%{[agent.version]}-*" +#setup.template.pattern: "{{.BeatIndexPrefix}}-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" diff --git a/libbeat/docs/howto/load-index-templates.asciidoc b/libbeat/docs/howto/load-index-templates.asciidoc index 885432a2671..f62efecdc04 100644 --- a/libbeat/docs/howto/load-index-templates.asciidoc +++ b/libbeat/docs/howto/load-index-templates.asciidoc @@ -224,7 +224,7 @@ PS > Invoke-RestMethod -Method Delete "http://localhost:9200/{beatname_lc}-*" ---------------------------------------------------------------------- endif::win_os[] -This command deletes all indices that match the pattern +{beat_default_index_prefix}-*+. +This command deletes all indices that match the pattern +{beat_default_index_prefix}+. Before running this command, make sure you want to delete all indices that match the pattern. diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 5699a46dd76..2ec64291b80 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -26,13 +26,6 @@ existing one. *`setup.template.enabled`*:: Set to false to disable template loading. If this is set to false, you must <>. -ifndef::apm-server[] -*`setup.template.type`*:: The type of template to use. Available options: `legacy` (default), index templates -before Elasticsearch v7.8. Use this to avoid breaking existing deployments. New options are `component` -and `index`. Selecting `component` loads a component template which can be included in new index templates. -The option `index` loads the new index template. -endif::[] - *`setup.template.name`*:: The name of the template. The default is +{beatname_lc}+. The {beatname_uc} version is always appended to the given name, so the final name is +{beatname_lc}-%{[{beat_version_key}]}+. @@ -44,17 +37,16 @@ name, so the final name is +{beatname_lc}-%{[{beat_version_key}]}+. // the example as expected. *`setup.template.pattern`*:: The template pattern to apply to the default index -settings. The default pattern is +{beat_default_index_prefix}-\*+. The {beatname_uc} version is always +settings. The default pattern is +{beat_default_index_prefix}+. The {beatname_uc} version is always included in the pattern, so the final pattern is -+{beat_default_index_prefix}-%{[{beat_version_key}]}-*+. The wildcard character `-*` is used to -match all daily indices. ++{beat_default_index_prefix}-%{[{beat_version_key}]}+. + Example: + ["source","yaml",subs="attributes"] ---------------------------------------------------------------------- setup.template.name: "{beatname_lc}" -setup.template.pattern: "{beat_default_index_prefix}-*" +setup.template.pattern: "{beat_default_index_prefix}" ---------------------------------------------------------------------- *`setup.template.fields`*:: The path to the YAML file describing the fields. The default is +fields.yml+. If a diff --git a/libbeat/idxmgmt/ilm/client_handler.go b/libbeat/idxmgmt/ilm/client_handler.go index 9ec4cc218d0..4d2163091ae 100644 --- a/libbeat/idxmgmt/ilm/client_handler.go +++ b/libbeat/idxmgmt/ilm/client_handler.go @@ -18,9 +18,7 @@ package ilm import ( - "encoding/json" "fmt" - "net/url" "path" "github.com/elastic/beats/v7/libbeat/common" @@ -29,10 +27,6 @@ import ( // ClientHandler defines the interface between a remote service and the Manager. type ClientHandler interface { CheckILMEnabled(bool) (bool, error) - - HasAlias(name string) (bool, error) - CreateAlias(alias Alias) error - HasILMPolicy(name string) (bool, error) CreateILMPolicy(policy Policy) error } @@ -42,10 +36,14 @@ type ESClientHandler struct { client ESClient } +type VersionCheckerClient interface { + GetVersion() common.Version +} + // ESClient defines the minimal interface required for the Loader to -// prepare a policy and write alias. +// prepare a policy. type ESClient interface { - GetVersion() common.Version + VersionCheckerClient Request( method, path string, pipeline string, @@ -60,24 +58,17 @@ type FileClientHandler struct { } // FileClient defines the minimal interface required for the Loader to -// prepare a policy and write alias. +// prepare a policy. type FileClient interface { GetVersion() common.Version Write(component string, name string, body string) error } const ( - // esFeaturesPath is used to query Elasticsearch for availability of licensed - // features. - esFeaturesPath = "/_xpack" - esILMPath = "/_ilm/policy" - - esAliasPath = "/_alias" ) var ( - esMinILMVersion = common.MustNewVersion("6.6.0") esMinDefaultILMVersion = common.MustNewVersion("7.0.0") ) @@ -93,31 +84,7 @@ func NewFileClientHandler(c FileClient) *FileClientHandler { // CheckILMEnabled indicates whether or not ILM is supported for the configured mode and ES instance. func (h *ESClientHandler) CheckILMEnabled(enabled bool) (bool, error) { - if !enabled { - return false, nil - } - - ver := h.client.GetVersion() - if !checkILMVersion(h.client.GetVersion()) { - return false, errf(ErrESVersionNotSupported, "Elasticsearch %v does not support ILM", ver.String()) - } - - avail, enabledILM, err := h.checkILMSupport() - if err != nil { - return false, err - } - - if !avail { - if enabledILM { - return false, errOf(ErrESVersionNotSupported) - } - return false, nil - } - - if !enabledILM && enabled { - return false, errOf(ErrESILMDisabled) - } - return enabled, nil + return checkILMEnabled(enabled, h.client) } // CreateILMPolicy loads the given policy to Elasticsearch. @@ -139,110 +106,21 @@ func (h *ESClientHandler) HasILMPolicy(name string) (bool, error) { return status == 200, nil } -// HasAlias queries Elasticsearch to see if alias exists. If other resource -// with the same name exists, it returns an error. -func (h *ESClientHandler) HasAlias(name string) (bool, error) { - status, b, err := h.client.Request("GET", esAliasPath+"/"+name, "", nil, nil) - if err != nil && status != 404 { - return false, wrapErrf(err, ErrRequestFailed, - "failed to check for alias '%v': (status=%v) %s", name, status, b) - } - if status == 200 { - return true, nil - } - - // Alias doesn't exist, check if there is an index with the same name - status, b, err = h.client.Request("HEAD", "/"+name, "", nil, nil) - if err != nil && status != 404 { - return false, wrapErrf(err, ErrRequestFailed, - "failed to check for alias '%v': (status=%v) %s", name, status, b) - } - if status == 200 { - return false, errf(ErrInvalidAlias, - "resource '%v' exists, but it is not an alias", name) - } - return false, nil -} - -// CreateAlias sends request to Elasticsearch for creating alias. -func (h *ESClientHandler) CreateAlias(alias Alias) error { - // Escaping because of date pattern - // This always assume it's a date pattern by sourrounding it by <...> - firstIndex := fmt.Sprintf("<%s-%s>", alias.Name, alias.Pattern) - firstIndex = url.PathEscape(firstIndex) - - body := common.MapStr{ - "aliases": common.MapStr{ - alias.Name: common.MapStr{ - "is_write_index": true, - }, - }, - } - - // Note: actual aliases are accessible via the index - if _, res, err := h.client.Request("PUT", "/"+firstIndex, "", nil, body); err != nil { - // Creating the index may fail for multiple reasons, e.g. because - // the index exists, or because the write alias exists and points - // to another index. - if ok, err := h.HasAlias(alias.Name); err != nil { - // HasAlias fails if there is an index with the same name. - return err - } else if ok { - return errOf(ErrAliasAlreadyExists) - } - return wrapErrf(err, ErrAliasCreateFailed, "failed to create alias: %s", res) - } - return nil -} - -func (h *ESClientHandler) checkILMSupport() (avail, enabled bool, err error) { - var response struct { - Features struct { - ILM struct { - Available bool `json:"available"` - Enabled bool `json:"enabled"` - } `json:"ilm"` - } `json:"features"` - } - status, err := h.queryFeatures(&response) - if status == 400 { - // If we get a 400, it's assumed to be the OSS version of Elasticsearch - return false, false, nil - } - if err != nil { - return false, false, wrapErr(err, ErrILMCheckRequestFailed) - } - - avail = response.Features.ILM.Available - enabled = response.Features.ILM.Enabled - return avail, enabled, nil -} - -func (h *ESClientHandler) queryFeatures(to interface{}) (int, error) { - status, body, err := h.client.Request("GET", esFeaturesPath, "", nil, nil) - if status >= 400 || err != nil { - return status, err - } - - if to != nil { - if err := json.Unmarshal(body, to); err != nil { - return status, wrapErrf(err, ErrInvalidResponse, "failed to parse JSON response") - } - } - return status, nil -} - // CheckILMEnabled indicates whether or not ILM is supported for the configured mode and client version. func (h *FileClientHandler) CheckILMEnabled(enabled bool) (bool, error) { + return checkILMEnabled(enabled, h.client) +} + +func checkILMEnabled(enabled bool, c VersionCheckerClient) (bool, error) { if !enabled { return false, nil } - version := h.client.GetVersion() - if checkILMVersion(version) { - return enabled, nil + + ver := c.GetVersion() + if ver.LessThan(esMinDefaultILMVersion) { + return false, errf(ErrESVersionNotSupported, "Elasticsearch %v does not support ILM", ver.String()) } - return false, errf(ErrESVersionNotSupported, - "Elasticsearch %v does not support ILM", version.String()) + return true, nil } // CreateILMPolicy writes given policy to the configured file. @@ -258,20 +136,3 @@ func (h *FileClientHandler) CreateILMPolicy(policy Policy) error { func (h *FileClientHandler) HasILMPolicy(name string) (bool, error) { return false, nil } - -// CreateAlias is a noop implementation. -func (h *FileClientHandler) CreateAlias(alias Alias) error { - return nil -} - -// HasAlias always returns false. -func (h *FileClientHandler) HasAlias(name string) (bool, error) { - return false, nil -} - -// avail: indicates whether version supports ILM -// probe: in case version potentially supports ILM, check the combination of mode + version -// to indicate whether or not ILM support should be enabled or disabled -func checkILMVersion(ver common.Version) (avail bool) { - return !ver.LessThan(esMinILMVersion) -} diff --git a/libbeat/idxmgmt/ilm/client_handler_integration_test.go b/libbeat/idxmgmt/ilm/client_handler_integration_test.go index b6f2755fd3d..42b19cf44ae 100644 --- a/libbeat/idxmgmt/ilm/client_handler_integration_test.go +++ b/libbeat/idxmgmt/ilm/client_handler_integration_test.go @@ -104,100 +104,6 @@ func TestESClientHandler_ILMPolicy(t *testing.T) { }) } -func TestESClientHandler_Alias(t *testing.T) { - makeAlias := func(base string) ilm.Alias { - return ilm.Alias{ - Name: makeName(base), - Pattern: "{now/d}-000001", - } - } - - t.Run("does not exist", func(t *testing.T) { - name := makeName("esch-alias-no") - h := newESClientHandler(t) - b, err := h.HasAlias(name) - assert.NoError(t, err) - assert.False(t, b) - }) - - t.Run("create new", func(t *testing.T) { - alias := makeAlias("esch-alias-create") - h := newESClientHandler(t) - err := h.CreateAlias(alias) - assert.NoError(t, err) - - b, err := h.HasAlias(alias.Name) - assert.NoError(t, err) - assert.True(t, b) - }) - - t.Run("create index exists", func(t *testing.T) { - alias := makeAlias("esch-alias-2create") - h := newESClientHandler(t) - - err := h.CreateAlias(alias) - assert.NoError(t, err) - - // Second time around creating the alias, ErrAliasAlreadyExists is returned: - // the initial index already exists and the write alias points to it. - err = h.CreateAlias(alias) - require.Error(t, err) - assert.Equal(t, ilm.ErrAliasAlreadyExists, ilm.ErrReason(err)) - - b, err := h.HasAlias(alias.Name) - assert.NoError(t, err) - assert.True(t, b) - }) - - t.Run("create alias exists", func(t *testing.T) { - alias := makeAlias("esch-alias-2create") - alias.Pattern = "000001" // no date math, so we get predictable index names - h := newESClientHandler(t) - - err := h.CreateAlias(alias) - assert.NoError(t, err) - - // Rollover, so write alias points at -000002. - es := newRawESClient(t) - _, _, err = es.Request("POST", "/"+alias.Name+"/_rollover", "", nil, nil) - require.NoError(t, err) - - // Delete -000001, simulating ILM delete. - _, _, err = es.Request("DELETE", "/"+alias.Name+"-"+alias.Pattern, "", nil, nil) - require.NoError(t, err) - - // Second time around creating the alias, ErrAliasAlreadyExists is returned: - // initial index does not exist, but the write alias exists and points to - // another index. - err = h.CreateAlias(alias) - require.Error(t, err) - assert.Equal(t, ilm.ErrAliasAlreadyExists, ilm.ErrReason(err)) - - b, err := h.HasAlias(alias.Name) - assert.NoError(t, err) - assert.True(t, b) - }) - - t.Run("resource exists but is not an alias", func(t *testing.T) { - alias := makeAlias("esch-alias-3create") - - es := newRawESClient(t) - - _, _, err := es.Request("PUT", "/"+alias.Name, "", nil, nil) - require.NoError(t, err) - - h := newESClientHandler(t) - - b, err := h.HasAlias(alias.Name) - assert.Equal(t, ilm.ErrInvalidAlias, ilm.ErrReason(err)) - assert.False(t, b) - - err = h.CreateAlias(alias) - require.Error(t, err) - assert.Equal(t, ilm.ErrInvalidAlias, ilm.ErrReason(err)) - }) -} - func newESClientHandler(t *testing.T) ilm.ClientHandler { client := newRawESClient(t) return ilm.NewESClientHandler(client) diff --git a/libbeat/idxmgmt/ilm/config.go b/libbeat/idxmgmt/ilm/config.go index f8e25f22f5b..ea5cd9ac1e4 100644 --- a/libbeat/idxmgmt/ilm/config.go +++ b/libbeat/idxmgmt/ilm/config.go @@ -18,8 +18,6 @@ package ilm import ( - "fmt" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" @@ -27,11 +25,9 @@ import ( // Config is used for unpacking a common.Config. type Config struct { - Enabled bool `config:"enabled"` - PolicyName fmtstr.EventFormatString `config:"policy_name"` - PolicyFile string `config:"policy_file"` - RolloverAlias fmtstr.EventFormatString `config:"rollover_alias"` - Pattern string `config:"pattern"` + Enabled bool `config:"enabled"` + PolicyName fmtstr.EventFormatString `config:"policy_name"` + PolicyFile string `config:"policy_file"` // CheckExists can disable the check for an existing policy. Check required // read_ilm privileges. If check is disabled the policy will only be @@ -42,8 +38,6 @@ type Config struct { Overwrite bool `config:"overwrite"` } -const ilmDefaultPattern = "{now/d}-000001" - // DefaultPolicy defines the default policy to be used if no custom policy is // configured. // By default the policy contains not warm, cold, or delete phase. @@ -65,23 +59,16 @@ var DefaultPolicy = common.MapStr{ //Validate verifies that expected config options are given and valid func (cfg *Config) Validate() error { - if cfg.RolloverAlias.IsEmpty() && cfg.Enabled { - return fmt.Errorf("rollover_alias must be set when ILM is not disabled") - } return nil } func defaultConfig(info beat.Info) Config { - name := info.Beat + "-%{[agent.version]}" - aliasFmt := fmtstr.MustCompileEvent(name) policyFmt := fmtstr.MustCompileEvent(info.Beat) return Config{ - Enabled: true, - PolicyName: *policyFmt, - RolloverAlias: *aliasFmt, - Pattern: ilmDefaultPattern, - PolicyFile: "", - CheckExists: true, + Enabled: true, + PolicyName: *policyFmt, + PolicyFile: "", + CheckExists: true, } } diff --git a/libbeat/idxmgmt/ilm/error.go b/libbeat/idxmgmt/ilm/error.go index a430ea87c4b..b2e9830afd7 100644 --- a/libbeat/idxmgmt/ilm/error.go +++ b/libbeat/idxmgmt/ilm/error.go @@ -36,9 +36,6 @@ var ( ErrInvalidResponse = errors.New("invalid response received") ErrESILMDisabled = errors.New("ILM is disabled in Elasticsearch") ErrRequestFailed = errors.New("request failed") - ErrAliasAlreadyExists = errors.New("alias already exists") - ErrAliasCreateFailed = errors.New("failed to create write alias") - ErrInvalidAlias = errors.New("invalid alias") ErrOpNotAvailable = errors.New("operation not available") ) diff --git a/libbeat/idxmgmt/ilm/ilm.go b/libbeat/idxmgmt/ilm/ilm.go index a9e64e3cbd9..30d4e4f888a 100644 --- a/libbeat/idxmgmt/ilm/ilm.go +++ b/libbeat/idxmgmt/ilm/ilm.go @@ -33,12 +33,11 @@ import ( // SupportFactory is used to define a policy type to be used. type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, error) -// Supporter implements ILM support. For loading the policies and creating -// write alias a manager instance must be generated. +// Supporter implements ILM support. For loading the policies +// a manager instance must be generated. type Supporter interface { // Query settings Enabled() bool - Alias() Alias Policy() Policy Overwrite() bool @@ -51,8 +50,6 @@ type Supporter interface { type Manager interface { CheckEnabled() (bool, error) - EnsureAlias() error - // EnsurePolicy installs a policy if it does not exist. The policy is always // written if overwrite is set. // The created flag is set to true only if a new policy is created. `created` @@ -67,12 +64,6 @@ type Policy struct { Body common.MapStr } -// Alias describes the alias to be created in Elasticsearch. -type Alias struct { - Name string - Pattern string -} - // DefaultSupport configures a new default ILM support implementation. func DefaultSupport(log *logp.Logger, info beat.Info, config *common.Config) (Supporter, error) { cfg := defaultConfig(info) @@ -109,16 +100,6 @@ func StdSupport(log *logp.Logger, info beat.Info, config *common.Config) (Suppor return nil, errors.Wrap(err, "failed to read ilm policy name") } - rolloverAlias, err := applyStaticFmtstr(info, &cfg.RolloverAlias) - if err != nil { - return nil, errors.Wrap(err, "failed to read the ilm rollover alias") - } - - alias := Alias{ - Name: rolloverAlias, - Pattern: cfg.Pattern, - } - policy := Policy{ Name: name, Body: DefaultPolicy, @@ -137,7 +118,7 @@ func StdSupport(log *logp.Logger, info beat.Info, config *common.Config) (Suppor policy.Body = body } - return NewStdSupport(log, cfg.Enabled, alias, policy, cfg.Overwrite, cfg.CheckExists), nil + return NewStdSupport(log, cfg.Enabled, policy, cfg.Overwrite, cfg.CheckExists), nil } // NoopSupport configures a new noop ILM support implementation, diff --git a/libbeat/idxmgmt/ilm/ilm_test.go b/libbeat/idxmgmt/ilm/ilm_test.go index 196b48e08ba..d6e33c2a892 100644 --- a/libbeat/idxmgmt/ilm/ilm_test.go +++ b/libbeat/idxmgmt/ilm/ilm_test.go @@ -31,28 +31,13 @@ import ( func TestDefaultSupport_Init(t *testing.T) { info := beat.Info{Beat: "test", Version: "9.9.9"} - t.Run("with an empty rollover_alias", func(t *testing.T) { - _, err := DefaultSupport(nil, info, common.MustNewConfigFrom( - map[string]interface{}{ - "enabled": true, - "rollover_alias": "", - "pattern": "01", - "check_exists": false, - "overwrite": true, - }, - )) - require.Error(t, err) - }) - t.Run("with custom config", func(t *testing.T) { tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom( map[string]interface{}{ - "enabled": true, - "name": "test-%{[agent.version]}", - "rollover_alias": "alias", - "pattern": "01", - "check_exists": false, - "overwrite": true, + "enabled": true, + "name": "test-%{[agent.version]}", + "check_exists": false, + "overwrite": true, }, )) require.NoError(t, err) @@ -63,17 +48,14 @@ func TestDefaultSupport_Init(t *testing.T) { assert.Equal(false, s.checkExists) assert.Equal(true, s.Enabled()) assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body)) - assert.Equal(Alias{Name: "alias", Pattern: "01"}, s.Alias()) }) t.Run("with custom alias config with fieldref", func(t *testing.T) { tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom( map[string]interface{}{ - "enabled": true, - "rollover_alias": "alias-%{[agent.version]}", - "pattern": "01", - "check_exists": false, - "overwrite": true, + "enabled": true, + "check_exists": false, + "overwrite": true, }, )) require.NoError(t, err) @@ -84,7 +66,6 @@ func TestDefaultSupport_Init(t *testing.T) { assert.Equal(false, s.checkExists) assert.Equal(true, s.Enabled()) assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body)) - assert.Equal(Alias{Name: "alias-9.9.9", Pattern: "01"}, s.Alias()) }) t.Run("with default alias", func(t *testing.T) { @@ -104,7 +85,6 @@ func TestDefaultSupport_Init(t *testing.T) { assert.Equal(false, s.checkExists) assert.Equal(true, s.Enabled()) assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body)) - assert.Equal(Alias{Name: "test-9.9.9", Pattern: "01"}, s.Alias()) }) t.Run("load external policy", func(t *testing.T) { @@ -146,13 +126,6 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { cfg: map[string]interface{}{"enabled": true}, enabled: true, }, - "fail enabled": { - calls: []onCall{ - onCheckILMEnabled(true).Return(false, nil), - }, - cfg: map[string]interface{}{"enabled": true}, - fail: ErrESILMDisabled, - }, "io error": { calls: []onCall{ onCheckILMEnabled(true).Return(false, errors.New("ups")), @@ -189,80 +162,6 @@ func TestDefaultSupport_Manager_Enabled(t *testing.T) { } } -func TestDefaultSupport_Manager_EnsureAlias(t *testing.T) { - alias := Alias{ - Name: "test-9.9.9", - Pattern: ilmDefaultPattern, - } - - cases := map[string]struct { - calls []onCall - cfg map[string]interface{} - fail error - }{ - "create new alias": { - calls: []onCall{ - onHasAlias(alias.Name).Return(false, nil), - onCreateAlias(alias).Return(nil), - }, - }, - "alias already exists": { - calls: []onCall{ - onHasAlias(alias.Name).Return(true, nil), - }, - }, - "fail": { - calls: []onCall{ - onHasAlias(alias.Name).Return(false, nil), - onCreateAlias(alias).Return(errOf(ErrRequestFailed)), - }, - fail: ErrRequestFailed, - }, - "overwrite non-existent": { - calls: []onCall{ - onCreateAlias(alias).Return(nil), - }, - fail: nil, - cfg: map[string]interface{}{"check_exists": true, "overwrite": true}, - }, - "try overwrite existing": { - calls: []onCall{ - onCreateAlias(alias).Return(errOf(ErrAliasAlreadyExists)), - }, - fail: nil, // we detect that that the alias exists, and call it a day. - cfg: map[string]interface{}{"check_exists": true, "overwrite": true}, - }, - "fail to overwrite": { - calls: []onCall{ - onCreateAlias(alias).Return(errOf(ErrAliasCreateFailed)), - }, - fail: ErrAliasCreateFailed, - cfg: map[string]interface{}{"check_exists": true, "overwrite": true}, - }, - } - - for name, test := range cases { - t.Run(name, func(t *testing.T) { - cfg := test.cfg - if cfg == nil { - cfg = map[string]interface{}{"alias": "test"} - } - - h := newMockHandler(test.calls...) - m := createManager(t, h, test.cfg) - err := m.EnsureAlias() - - if test.fail == nil { - require.NoError(t, err) - } else { - require.Error(t, err) - assert.Equal(t, test.fail, ErrReason(err)) - } - h.AssertExpectations(t) - }) - } -} - func TestDefaultSupport_Manager_EnsurePolicy(t *testing.T) { testPolicy := Policy{ Name: "test", diff --git a/libbeat/idxmgmt/ilm/mockapihandler_test.go b/libbeat/idxmgmt/ilm/mockapihandler_test.go index 79148d83f1b..b527ad118d7 100644 --- a/libbeat/idxmgmt/ilm/mockapihandler_test.go +++ b/libbeat/idxmgmt/ilm/mockapihandler_test.go @@ -50,18 +50,6 @@ func (h *mockHandler) CheckILMEnabled(enabled bool) (bool, error) { return args.Bool(0), args.Error(1) } -func onHasAlias(name string) onCall { return makeOnCall("HasAlias", name) } -func (h *mockHandler) HasAlias(name string) (bool, error) { - args := h.Called(name) - return args.Bool(0), args.Error(1) -} - -func onCreateAlias(alias Alias) onCall { return makeOnCall("CreateAlias", alias) } -func (h *mockHandler) CreateAlias(alias Alias) error { - args := h.Called(alias) - return args.Error(0) -} - func onHasILMPolicy(name string) onCall { return makeOnCall("HasILMPolicy", name) } func (h *mockHandler) HasILMPolicy(name string) (bool, error) { args := h.Called(name) diff --git a/libbeat/idxmgmt/ilm/noop.go b/libbeat/idxmgmt/ilm/noop.go index 20586a261d3..a06c173d567 100644 --- a/libbeat/idxmgmt/ilm/noop.go +++ b/libbeat/idxmgmt/ilm/noop.go @@ -26,17 +26,15 @@ type noopSupport struct{} type noopManager struct{} // NewNoopSupport creates a noop ILM implementation with ILM support being always -// disabled. Attempts to install a policy or create a write alias will fail. +// disabled. Attempts to install a policy will fail. func NewNoopSupport(info beat.Info, config *common.Config) (Supporter, error) { return (*noopSupport)(nil), nil } func (*noopSupport) Enabled() bool { return false } -func (*noopSupport) Alias() Alias { return Alias{} } func (*noopSupport) Policy() Policy { return Policy{} } func (*noopSupport) Overwrite() bool { return false } func (*noopSupport) Manager(_ ClientHandler) Manager { return (*noopManager)(nil) } func (*noopManager) CheckEnabled() (bool, error) { return false, nil } -func (*noopManager) EnsureAlias() error { return errOf(ErrOpNotAvailable) } func (*noopManager) EnsurePolicy(_ bool) (bool, error) { return false, errOf(ErrOpNotAvailable) } diff --git a/libbeat/idxmgmt/ilm/std.go b/libbeat/idxmgmt/ilm/std.go index 6363898a0a1..ef8a11f7f21 100644 --- a/libbeat/idxmgmt/ilm/std.go +++ b/libbeat/idxmgmt/ilm/std.go @@ -30,7 +30,6 @@ type stdSupport struct { overwrite bool checkExists bool - alias Alias policy Policy } @@ -53,7 +52,6 @@ var defaultCacheDuration = 5 * time.Minute func NewStdSupport( log *logp.Logger, enabled bool, - alias Alias, policy Policy, overwrite, checkExists bool, ) Supporter { @@ -62,13 +60,11 @@ func NewStdSupport( enabled: enabled, overwrite: overwrite, checkExists: checkExists, - alias: alias, policy: policy, } } func (s *stdSupport) Enabled() bool { return s.enabled } -func (s *stdSupport) Alias() Alias { return s.alias } func (s *stdSupport) Policy() Policy { return s.policy } func (s *stdSupport) Overwrite() bool { return s.overwrite } @@ -93,59 +89,11 @@ func (m *stdManager) CheckEnabled() (bool, error) { return ilmEnabled, err } - if !ilmEnabled && m.enabled { - return false, errOf(ErrESILMDisabled) - } - m.cache.Enabled = ilmEnabled m.cache.LastUpdate = time.Now() return ilmEnabled, nil } -func (m *stdManager) EnsureAlias() error { - log := m.log - if !m.checkExists { - log.Infof("Index alias is not checked as setup.ilm.check_exists is disabled") - return nil - } - - overwrite := m.Overwrite() - name := m.alias.Name - - var exists bool - if !overwrite { - var err error - exists, err = m.client.HasAlias(name) - if err != nil { - return err - } - } - - switch { - case exists && !overwrite: - log.Infof("Index Alias %v exists already.", name) - return nil - - case !exists || overwrite: - err := m.client.CreateAlias(m.alias) - if err != nil { - if ErrReason(err) != ErrAliasAlreadyExists { - log.Errorf("Index Alias %v setup failed: %v.", name, err) - return err - } - log.Infof("Index Alias %v exists already.", name) - return nil - } - - log.Infof("Index Alias %v successfully created.", name) - return nil - - default: - m.log.Infof("ILM index alias not created: exists=%v, overwrite=%v", exists, overwrite) - return nil - } -} - func (m *stdManager) EnsurePolicy(overwrite bool) (bool, error) { log := m.log if !m.checkExists { diff --git a/libbeat/idxmgmt/mockilm_test.go b/libbeat/idxmgmt/mockilm_test.go index d138b46e6a0..05ebbe87353 100644 --- a/libbeat/idxmgmt/mockilm_test.go +++ b/libbeat/idxmgmt/mockilm_test.go @@ -57,12 +57,6 @@ func (m *mockILMSupport) Enabled() bool { return args.Get(0).(bool) } -func onAlias() onCall { return makeOnCall("Alias") } -func (m *mockILMSupport) Alias() ilm.Alias { - args := m.Called() - return args.Get(0).(ilm.Alias) -} - func onPolicy() onCall { return makeOnCall("Policy") } func (m *mockILMSupport) Policy() ilm.Policy { args := m.Called() @@ -84,12 +78,6 @@ func (m *mockILMSupport) CheckEnabled() (bool, error) { return args.Bool(0), args.Error(1) } -func onEnsureAlias() onCall { return makeOnCall("EnsureAlias") } -func (m *mockILMSupport) EnsureAlias() error { - args := m.Called() - return args.Error(0) -} - func onEnsurePolicy() onCall { return makeOnCall("EnsurePolicy") } func (m *mockILMSupport) EnsurePolicy(overwrite bool) (bool, error) { args := m.Called() diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 6cca4dad683..e32fdfd647c 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -63,7 +63,6 @@ type indexSelector struct { type ilmIndexSelector struct { index outil.Selector - alias outil.Selector st *indexState beatInfo beat.Info } @@ -112,7 +111,7 @@ func newIndexSupport( return nil, err } - tmplCfg, err := unpackTemplateConfig(tmplConfig) + tmplCfg, err := unpackTemplateConfig(info, tmplConfig) if err != nil { return nil, err } @@ -123,7 +122,7 @@ func newIndexSupport( info: info, templateCfg: tmplCfg, migration: migration, - defaultIndex: fmt.Sprintf("%v-%v-%%{+yyyy.MM.dd}", info.IndexPrefix, info.Version), + defaultIndex: fmt.Sprintf("%v-%v", info.IndexPrefix, info.Version), }, nil } @@ -155,8 +154,6 @@ func (s *indexSupport) Manager( func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, error) { var err error - log := s.log - // we construct our own configuration object based on the available settings // in cfg and defaultIndex. The configuration object provided must not be // modified. @@ -177,12 +174,6 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, } } - if s.ilm.Enabled() { - alias := s.ilm.Alias().Name - log.Infof("Set %v to '%s' as ILM is enabled.", cfg.PathOf("index"), alias) - indexName = alias - } - // no index name configuration found yet -> define default index name based on // beat.Info provided to the indexSupport on during setup. if indexName == "" { @@ -213,19 +204,18 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string m.support.templateCfg.Overwrite, loadTemplate) if ilmComponent.load && !templateComponent.load { - return false, "Loading ILM policy and write alias without loading template " + - "is not recommended. Check your configuration." + return false, "Loading ILM policy without loading template is not recommended. Check your configuration." } if templateComponent.load && !ilmComponent.load && ilmComponent.enabled { return false, "Loading template with ILM settings whithout loading ILM " + - "policy and alias can lead to issues and is not recommended. " + + "policy can lead to issues and is not recommended. " + "Check your configuration." } var warn string if !ilmComponent.load { - warn += "ILM policy and write alias loading not enabled.\n" + warn += "ILM policy loading not enabled.\n" } else if !ilmComponent.overwrite { warn += "Overwriting ILM policy is disabled. Set `setup.ilm.overwrite: true` for enabling.\n" } @@ -243,7 +233,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { if err != nil { return err } - if withILM && loadILM.Enabled() { + if withILM { log.Info("Auto ILM enable success.") } @@ -269,7 +259,7 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled if ilmComponent.enabled { - tmplCfg, err = applyILMSettings(log, tmplCfg, m.support.ilm.Policy(), m.support.ilm.Alias()) + tmplCfg, err = applyILMSettings(log, tmplCfg, m.support.ilm.Policy()) if err != nil { return err } @@ -283,13 +273,6 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Loaded index template.") } - if ilmComponent.load { - err := m.ilm.EnsureAlias() - if err != nil { - return err - } - } - return nil } @@ -314,11 +297,6 @@ func (s *ilmIndexSelector) Select(evt *beat.Event) (string, error) { return idx, nil } - if s.st.withILM.Load() { - idx, err := s.alias.Select(evt) - return idx, err - } - idx, err := s.index.Select(evt) return idx, err } @@ -335,10 +313,6 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string { return "" } - if alias, err := events.GetMetaStringValue(*evt, events.FieldMetaAlias); err == nil { - return strings.ToLower(alias) - } - if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaIndex); err == nil { ts := evt.Timestamp.UTC() return fmt.Sprintf("%s-%d.%02d.%02d", @@ -356,8 +330,9 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string { return "" } -func unpackTemplateConfig(cfg *common.Config) (config template.TemplateConfig, err error) { - config = template.DefaultConfig() +func unpackTemplateConfig(info beat.Info, cfg *common.Config) (config template.TemplateConfig, err error) { + config = template.DefaultConfig(info) + if cfg != nil { err = cfg.Unpack(&config) } @@ -368,32 +343,15 @@ func applyILMSettings( log *logp.Logger, tmpl template.TemplateConfig, policy ilm.Policy, - alias ilm.Alias, ) (template.TemplateConfig, error) { if !tmpl.Enabled { return tmpl, nil } - if alias.Name == "" { - return tmpl, errors.New("no ilm rollover alias configured") - } - if policy.Name == "" { return tmpl, errors.New("no ilm policy name configured") } - tmpl.Name = alias.Name - if log != nil { - log.Infof("Set setup.template.name to '%s' as ILM is enabled.", alias) - } - - tmpl.Pattern = fmt.Sprintf("%s-*", alias.Name) - if log != nil { - log.Infof("Set setup.template.pattern to '%s' as ILM is enabled.", tmpl.Pattern) - } - - // rollover_alias and lifecycle.name can't be configured and will be overwritten - // init/copy index settings idxSettings := tmpl.Settings.Index if idxSettings == nil { @@ -421,11 +379,6 @@ func applyILMSettings( } idxSettings["lifecycle"] = lifecycle - // add rollover_alias and name to index.lifecycle settings - if _, exists := lifecycle["rollover_alias"]; !exists { - log.Infof("Set settings.index.lifecycle.rollover_alias in template to %s as ILM is enabled.", alias) - lifecycle["rollover_alias"] = alias.Name - } if _, exists := lifecycle["name"]; !exists { log.Infof("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", policy) lifecycle["name"] = policy.Name diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index 76b0ac9879e..4a7a4c848cb 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -33,7 +33,7 @@ import ( ) type mockClientHandler struct { - alias, policy string + policy string expectsPolicy bool tmplCfg *template.TemplateConfig @@ -47,7 +47,6 @@ type mockCreateOp uint8 const ( mockCreatePolicy mockCreateOp = iota mockCreateTemplate - mockCreateAlias ) func TestDefaultSupport_Enabled(t *testing.T) { @@ -99,10 +98,9 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { type nameFunc func(time.Time) string noILM := []onCall{onEnabled().Return(false)} - ilmTemplateSettings := func(alias, policy string) []onCall { + ilmTemplateSettings := func(policy string) []onCall { return []onCall{ onEnabled().Return(true), - onAlias().Return(ilm.Alias{Name: alias}), onPolicy().Return(ilm.Policy{Name: policy}), } } @@ -135,22 +133,6 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { cfg: map[string]interface{}{"index": "TeSt-%{[agent.version]}"}, want: stable("test-9.9.9"), }, - "event alias without ilm": { - ilmCalls: noILM, - cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, - want: stable("test"), - meta: common.MapStr{ - "alias": "test", - }, - }, - "event alias without ilm must be lowercae": { - ilmCalls: noILM, - cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, - want: stable("test"), - meta: common.MapStr{ - "alias": "Test", - }, - }, "event index without ilm": { ilmCalls: noILM, cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -168,33 +150,17 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { }, }, "with ilm": { - ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), - cfg: map[string]interface{}{"index": "wrong-%{[agent.version]}"}, + ilmCalls: ilmTemplateSettings("test-9.9.9"), + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, want: stable("test-9.9.9"), }, "with ilm must be lowercase": { - ilmCalls: ilmTemplateSettings("Test-9.9.9", "Test-9.9.9"), - cfg: map[string]interface{}{"index": "wrong-%{[agent.version]}"}, - want: stable("test-9.9.9"), - }, - "event alias wit ilm": { - ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + ilmCalls: ilmTemplateSettings("Test-9.9.9"), cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, - want: stable("event-alias"), - meta: common.MapStr{ - "alias": "event-alias", - }, - }, - "event alias wit ilm must be lowercase": { - ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), - cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, - want: stable("event-alias"), - meta: common.MapStr{ - "alias": "Event-alias", - }, + want: stable("test-9.9.9"), }, "event index with ilm": { - ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + ilmCalls: ilmTemplateSettings("test-9.9.9"), cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, want: dateIdx("event-index"), meta: common.MapStr{ @@ -202,7 +168,7 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { }, }, "use indices": { - ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + ilmCalls: ilmTemplateSettings("test-9.9.9"), cfg: map[string]interface{}{ "index": "test-%{[agent.version]}", "indices": []map[string]interface{}{ @@ -212,7 +178,7 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { want: stable("myindex"), }, "use indices settings must be lowercase": { - ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + ilmCalls: ilmTemplateSettings("test-9.9.9"), cfg: map[string]interface{}{ "index": "test-%{[agent.version]}", "indices": []map[string]interface{}{ @@ -260,7 +226,7 @@ func TestIndexManager_VerifySetup(t *testing.T) { }{ "load template with ilm without loading ilm": { ilmEnabled: true, tmplEnabled: true, loadILM: LoadModeDisabled, - warn: "whithout loading ILM policy and alias", + warn: "whithout loading ILM policy", }, "load ilm without template": { ilmEnabled: true, loadILM: LoadModeUnset, @@ -346,40 +312,36 @@ func TestIndexManager_Setup(t *testing.T) { } return &s } - defaultCfg := template.DefaultConfig() + info := beat.Info{Beat: "test", Version: "9.9.9"} + defaultCfg := template.DefaultConfig(info) cases := map[string]struct { cfg common.MapStr loadTemplate, loadILM LoadMode - err bool - tmplCfg *template.TemplateConfig - alias, policy string + err bool + tmplCfg *template.TemplateConfig + policy string }{ "template default ilm default": { - tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ + tmplCfg: cfgWith(template.DefaultConfig(info), map[string]interface{}{ "overwrite": "true", "name": "test-9.9.9", - "pattern": "test-9.9.9-*", + "pattern": "test-9.9.9", "settings.index.lifecycle.name": "test", - "settings.index.lifecycle.rollover_alias": "test-9.9.9", }), - alias: "test-9.9.9", policy: "test", }, - "template default ilm default with alias and policy changed": { + "template default ilm default with policy changed": { cfg: common.MapStr{ - "setup.ilm.rollover_alias": "mocktest", - "setup.ilm.policy_name": "policy-keep", + "setup.ilm.policy_name": "policy-keep", }, - tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ + tmplCfg: cfgWith(template.DefaultConfig(info), map[string]interface{}{ "overwrite": "true", - "name": "mocktest", - "pattern": "mocktest-*", + "name": "test-9.9.9", + "pattern": "test-9.9.9", "settings.index.lifecycle.name": "policy-keep", - "settings.index.lifecycle.rollover_alias": "mocktest", }), - alias: "mocktest", policy: "policy-keep", }, "template default ilm disabled": { @@ -394,16 +356,20 @@ func TestIndexManager_Setup(t *testing.T) { "setup.ilm.enabled": false, }, loadTemplate: LoadModeOverwrite, - tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ + tmplCfg: cfgWith(template.DefaultConfig(info), map[string]interface{}{ "overwrite": "true", + "name": "test-9.9.9", + "pattern": "test-9.9.9", }), }, "template default loadMode Force ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, + "name": "test-9.9.9", + "pattern": "test-9.9.9", }, loadTemplate: LoadModeForce, - tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ + tmplCfg: cfgWith(template.DefaultConfig(info), map[string]interface{}{ "overwrite": "true", }), }, @@ -417,7 +383,6 @@ func TestIndexManager_Setup(t *testing.T) { cfg: common.MapStr{ "setup.template.enabled": false, }, - alias: "test-9.9.9", policy: "test", }, "template disabled ilm disabled, loadMode Overwrite": { @@ -433,22 +398,19 @@ func TestIndexManager_Setup(t *testing.T) { "setup.ilm.enabled": false, }, loadILM: LoadModeForce, - alias: "test-9.9.9", policy: "test", }, "template loadmode disabled ilm loadMode enabled": { loadTemplate: LoadModeDisabled, loadILM: LoadModeEnabled, - alias: "test-9.9.9", policy: "test", }, "template default ilm loadMode disabled": { loadILM: LoadModeDisabled, - tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ + tmplCfg: cfgWith(template.DefaultConfig(info), map[string]interface{}{ "name": "test-9.9.9", - "pattern": "test-9.9.9-*", + "pattern": "test-9.9.9", "settings.index.lifecycle.name": "test", - "settings.index.lifecycle.rollover_alias": "test-9.9.9", }), }, "template loadmode disabled ilm loadmode disabled": { @@ -458,7 +420,6 @@ func TestIndexManager_Setup(t *testing.T) { } for name, test := range cases { t.Run(name, func(t *testing.T) { - info := beat.Info{Beat: "test", Version: "9.9.9"} factory := MakeDefaultSupport(ilm.StdSupport) im, err := factory(nil, info, common.MustNewConfigFrom(test.cfg)) require.NoError(t, err) @@ -477,7 +438,6 @@ func TestIndexManager_Setup(t *testing.T) { } else { assert.Equal(t, test.tmplCfg, clientHandler.tmplCfg) } - assert.Equal(t, test.alias, clientHandler.alias) assert.Equal(t, test.policy, clientHandler.policy) } }) @@ -485,7 +445,7 @@ func TestIndexManager_Setup(t *testing.T) { } func (op mockCreateOp) String() string { - names := []string{"create-policy", "create-template", "create-alias"} + names := []string{"create-policy", "create-template"} if int(op) > len(names) { return "unknown" } @@ -507,16 +467,6 @@ func (h *mockClientHandler) CheckILMEnabled(enabled bool) (bool, error) { return enabled, nil } -func (h *mockClientHandler) HasAlias(name string) (bool, error) { - return h.alias == name, nil -} - -func (h *mockClientHandler) CreateAlias(alias ilm.Alias) error { - h.recordOp(mockCreateAlias) - h.alias = alias.Name - return nil -} - func (h *mockClientHandler) HasILMPolicy(name string) (bool, error) { return h.policy == name, nil } diff --git a/libbeat/publisher/processing/processors.go b/libbeat/publisher/processing/processors.go index 19111b6aff9..cac9d356153 100644 --- a/libbeat/publisher/processing/processors.go +++ b/libbeat/publisher/processing/processors.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/joeshaw/multierror" @@ -51,6 +52,11 @@ func newGeneralizeProcessor(keepNull bool) *processorFn { return nil, nil } + // data streams require @timestamp field + if event.Timestamp.IsZero() { + event.Timestamp = time.Now() + } + fields := g.Convert(event.Fields) if fields == nil { logger.Error("fail to convert to generic event") diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 7eb6ff522f4..4f9fe2348c7 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -18,27 +18,10 @@ package template import ( - "fmt" - + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/mapping" ) -const ( - IndexTemplateLegacy IndexTemplateType = iota - IndexTemplateComponent - IndexTemplateIndex -) - -var ( - templateTypes = map[string]IndexTemplateType{ - "legacy": IndexTemplateLegacy, - "component": IndexTemplateComponent, - "index": IndexTemplateIndex, - } -) - -type IndexTemplateType uint8 - // TemplateConfig holds config information about the Elasticsearch template type TemplateConfig struct { Enabled bool `config:"enabled"` @@ -50,12 +33,10 @@ type TemplateConfig struct { Path string `config:"path"` Name string `config:"name"` } `config:"json"` - AppendFields mapping.Fields `config:"append_fields"` - Overwrite bool `config:"overwrite"` - Settings TemplateSettings `config:"settings"` - Order int `config:"order"` - Priority int `config:"priority"` - Type IndexTemplateType `config:"type"` + AppendFields mapping.Fields `config:"append_fields"` + Overwrite bool `config:"overwrite"` + Settings TemplateSettings `config:"settings"` + Priority int `config:"priority"` } // TemplateSettings are part of the Elasticsearch template and hold index and source specific information. @@ -65,28 +46,12 @@ type TemplateSettings struct { } // DefaultConfig for index template -func DefaultConfig() TemplateConfig { +func DefaultConfig(info beat.Info) TemplateConfig { return TemplateConfig{ + Name: info.Beat + "-" + info.Version, + Pattern: info.Beat + "-" + info.Version, Enabled: true, Fields: "", - Type: IndexTemplateLegacy, - Order: 1, Priority: 150, } } - -func (t *IndexTemplateType) Unpack(v string) error { - if v == "" { - *t = IndexTemplateLegacy - return nil - } - - var tt IndexTemplateType - var ok bool - if tt, ok = templateTypes[v]; !ok { - return fmt.Errorf("unknown index template type: %s", v) - } - *t = tt - - return nil -} diff --git a/libbeat/template/load.go b/libbeat/template/load.go index 0ecbc291268..1900d8f5a75 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -24,7 +24,6 @@ import ( "io/ioutil" "net/http" "os" - "strings" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -32,14 +31,6 @@ import ( "github.com/elastic/beats/v7/libbeat/paths" ) -var ( - templateLoaderPath = map[IndexTemplateType]string{ - IndexTemplateLegacy: "/_template/", - IndexTemplateComponent: "/_component_template/", - IndexTemplateIndex: "/_index_template/", - } -) - // Loader interface for loading templates. type Loader interface { Load(config TemplateConfig, info beat.Info, fields []byte, migration bool) error @@ -114,7 +105,7 @@ func (l *ESLoader) Load(config TemplateConfig, info beat.Info, fields []byte, mi templateName = config.JSON.Name } - exists, err := l.templateExists(templateName, config.Type) + exists, err := l.checkExistsTemplate(templateName) if err != nil { return fmt.Errorf("failure while checking if template exists: %w", err) } @@ -129,7 +120,7 @@ func (l *ESLoader) Load(config TemplateConfig, info beat.Info, fields []byte, mi if err != nil { return err } - if err := l.loadTemplate(templateName, config.Type, body); err != nil { + if err := l.loadTemplate(templateName, body); err != nil { return fmt.Errorf("failed to load template: %w", err) } l.log.Infof("Template with name %q loaded.", templateName) @@ -139,12 +130,10 @@ func (l *ESLoader) Load(config TemplateConfig, info beat.Info, fields []byte, mi // loadTemplate loads a template into Elasticsearch overwriting the existing // template if it exists. If you wish to not overwrite an existing template // then use CheckTemplate prior to calling this method. -func (l *ESLoader) loadTemplate(templateName string, templateType IndexTemplateType, template map[string]interface{}) error { +func (l *ESLoader) loadTemplate(templateName string, template map[string]interface{}) error { l.log.Infof("Try loading template %s to Elasticsearch", templateName) - clientVersion := l.client.GetVersion() - path := templateLoaderPath[templateType] + templateName - params := esVersionParams(clientVersion) - status, body, err := l.client.Request("PUT", path, "", params, template) + path := "/_index_template/" + templateName + status, body, err := l.client.Request("PUT", path, "", nil, template) if err != nil { return fmt.Errorf("couldn't load template: %v. Response body: %s", err, body) } @@ -154,52 +143,21 @@ func (l *ESLoader) loadTemplate(templateName string, templateType IndexTemplateT return nil } -func (l *ESLoader) templateExists(templateName string, templateType IndexTemplateType) (bool, error) { - if templateType == IndexTemplateComponent { - return l.checkExistsComponentTemplate(templateName) - } - return l.checkExistsTemplate(templateName) -} - // existsTemplate checks if a given template already exist, using the -// `_cat/templates/` API. +// `/_index_template/` API. // // An error is returned if the loader failed to execute the request, or a // status code indicating some problems is encountered. func (l *ESLoader) checkExistsTemplate(name string) (bool, error) { - status, body, err := l.client.Request("GET", "/_cat/templates/"+name, "", nil, nil) + status, _, err := l.client.Request("HEAD", "/_index_template/"+name, "", nil, nil) + if status == http.StatusNotFound { + return false, nil + } if err != nil { return false, err } - // Elasticsearch API returns 200, even if the template does not exists. We - // need to validate the body to be sure the template is actually known. Any - // status code other than 200 will be treated as error. - if status != http.StatusOK { - return false, &StatusError{status: status} - } - return strings.Contains(string(body), name), nil -} - -// existsComponentTemplate checks if a component template exists by querying -// the `_component_template/` API. -// -// The resource is assumed as present if a 200 OK status is returned and missing if a 404 is returned. -// Other status codes or IO errors during the request are reported as error. -func (l *ESLoader) checkExistsComponentTemplate(name string) (bool, error) { - status, _, err := l.client.Request("GET", "/_component_template/"+name, "", nil, nil) - - switch status { - case http.StatusNotFound: - return false, nil - case http.StatusOK: - return true, nil - default: - if err == nil { - err = &StatusError{status: status} - } - return false, err - } + return true, nil } // Load reads the template from the config, creates the template body and prints it to the configured file. @@ -247,7 +205,8 @@ func (b *templateBuilder) buildBody(tmpl *Template, config TemplateConfig, field return b.buildBodyFromFile(tmpl, config) } if fields == nil { - return b.buildMinimalTemplate(tmpl) + b.log.Debug("Load minimal template") + return tmpl.LoadMinimal(), nil } return b.buildBodyFromFields(tmpl, fields) } @@ -290,25 +249,6 @@ func (b *templateBuilder) buildBodyFromFields(tmpl *Template, fields []byte) (co return body, nil } -func (b *templateBuilder) buildMinimalTemplate(tmpl *Template) (common.MapStr, error) { - b.log.Debug("Load minimal template") - body, err := tmpl.LoadMinimal() - if err != nil { - return nil, fmt.Errorf("error creating mimimal template: %v", err) - } - return body, nil -} - func (e *StatusError) Error() string { return fmt.Sprintf("request failed with http status code %v", e.status) } - -func esVersionParams(ver common.Version) map[string]string { - if ver.Major == 6 && ver.Minor == 7 { - return map[string]string{ - "include_type_name": "true", - } - } - - return nil -} diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 371991810e4..07caf39c3e7 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -67,13 +67,13 @@ func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { t.Fatal(err) } s := testSetup{t: t, client: client, loader: NewESLoader(client), config: cfg} - client.Request("DELETE", templateLoaderPath[cfg.Type]+cfg.Name, "", nil, nil) + client.Request("DELETE", "/_index_template/"+cfg.Name, "", nil, nil) s.requireTemplateDoesNotExist("") return &s } func (ts *testSetup) mustLoadTemplate(body map[string]interface{}) { - err := ts.loader.loadTemplate(ts.config.Name, ts.config.Type, body) + err := ts.loader.loadTemplate(ts.config.Name, body) require.NoError(ts.t, err) ts.requireTemplateExists("") } @@ -103,16 +103,21 @@ func (ts *testSetup) requireTemplateExists(name string) { if name == "" { name = ts.config.Name } - exists, err := ts.loader.templateExists(name, ts.config.Type) + exists, err := ts.loader.checkExistsTemplate(name) require.NoError(ts.t, err, "failed to query template status") - require.True(ts.t, exists, "template must exist") + require.True(ts.t, exists, "template must exist: %s", name) +} + +func (ts *testSetup) cleanupTemplate(name string) { + ts.client.Request("DELETE", "/_index_template/"+name, "", nil, nil) + ts.requireTemplateDoesNotExist(name) } func (ts *testSetup) requireTemplateDoesNotExist(name string) { if name == "" { name = ts.config.Name } - exists, err := ts.loader.templateExists(name, ts.config.Type) + exists, err := ts.loader.checkExistsTemplate(name) require.NoError(ts.t, err, "failed to query template status") require.False(ts.t, exists, "template must not exist") } @@ -146,14 +151,14 @@ func TestESLoader_Load(t *testing.T) { t.Run("disabled", func(t *testing.T) { setup.load(nil) - tmpl := getTemplate(t, setup.client, setup.config.Name, setup.config.Type) + tmpl := getTemplate(t, setup.client, setup.config.Name) assert.Equal(t, true, tmpl.SourceEnabled()) }) t.Run("enabled", func(t *testing.T) { setup.config.Overwrite = true setup.load(nil) - tmpl := getTemplate(t, setup.client, setup.config.Name, setup.config.Type) + tmpl := getTemplate(t, setup.client, setup.config.Name) assert.Equal(t, false, tmpl.SourceEnabled()) }) }) @@ -172,6 +177,7 @@ func TestESLoader_Load(t *testing.T) { }{Enabled: true, Path: path(t, []string{"testdata", "fields.json"}), Name: nameJSON} setup.load(nil) setup.requireTemplateExists(nameJSON) + setup.cleanupTemplate(nameJSON) }) t.Run("load template successful", func(t *testing.T) { @@ -188,26 +194,17 @@ func TestESLoader_Load(t *testing.T) { fields: fields, properties: []string{"foo", "bar"}, }, - "default config with fields and component": { - cfg: TemplateConfig{Enabled: true, Type: IndexTemplateComponent}, - fields: fields, - properties: []string{"foo", "bar"}, - }, "minimal template": { cfg: TemplateConfig{Enabled: true}, fields: nil, }, - "minimal template component": { - cfg: TemplateConfig{Enabled: true, Type: IndexTemplateComponent}, - fields: nil, - }, "fields from file": { cfg: TemplateConfig{Enabled: true, Fields: path(t, []string{"testdata", "fields.yml"})}, fields: fields, - properties: []string{"object", "keyword", "alias", "migration_alias_false", "object_disabled"}, + properties: []string{"object", "keyword", "alias", "migration_alias_false", "object_disabled", "@timestamp"}, }, "fields from json": { - cfg: TemplateConfig{Enabled: true, Name: "json-template", JSON: struct { + cfg: TemplateConfig{Enabled: true, JSON: struct { Enabled bool `config:"enabled"` Path string `config:"path"` Name string `config:"name"` @@ -217,12 +214,15 @@ func TestESLoader_Load(t *testing.T) { }, } { t.Run(run, func(t *testing.T) { + if data.cfg.JSON.Enabled { + data.cfg.Name = data.cfg.JSON.Name + } setup := newTestSetup(t, data.cfg) setup.mustLoad(data.fields) // Fetch properties - tmpl := getTemplate(t, setup.client, setup.config.Name, setup.config.Type) - val, err := tmpl.GetValue("mappings.properties") + tmpl := getTemplate(t, setup.client, setup.config.Name) + val, err := tmpl.GetValue("template.mappings.properties") if data.properties == nil { assert.Error(t, err) } else { @@ -235,6 +235,7 @@ func TestESLoader_Load(t *testing.T) { } assert.ElementsMatch(t, properties, data.properties) } + setup.cleanupTemplate(setup.config.Name) }) } }) @@ -250,7 +251,7 @@ func TestLoadInvalidTemplate(t *testing.T) { // Try to load invalid template template := map[string]interface{}{"json": "invalid"} - err := setup.loader.loadTemplate(setup.config.Name, setup.config.Type, template) + err := setup.loader.loadTemplate(setup.config.Name, template) assert.Error(t, err) setup.requireTemplateDoesNotExist("") } @@ -276,7 +277,7 @@ func TestTemplateSettings(t *testing.T) { setup.mustLoadFromFile([]string{"..", "fields.yml"}) // Check that it contains the mapping - templateJSON := getTemplate(t, setup.client, setup.config.Name, setup.config.Type) + templateJSON := getTemplate(t, setup.client, setup.config.Name) assert.Equal(t, 1, templateJSON.NumberOfShards()) assert.Equal(t, false, templateJSON.SourceEnabled()) } @@ -287,8 +288,9 @@ var dataTests = []struct { }{ { data: common.MapStr{ - "keyword": "test keyword", - "array": [...]int{1, 2, 3}, + "@timestamp": time.Now(), + "keyword": "test keyword", + "array": [...]int{1, 2, 3}, "object": common.MapStr{ "hello": "world", }, @@ -314,6 +316,7 @@ var dataTests = []struct { { // tests enabled: false values data: common.MapStr{ + "@timestamp": time.Now(), "array_disabled": [...]int{1, 2, 3}, "object_disabled": common.MapStr{ "hello": "world", @@ -340,8 +343,8 @@ func TestTemplateWithData(t *testing.T) { } } -func getTemplate(t *testing.T, client ESClient, templateName string, templateType IndexTemplateType) testTemplate { - status, body, err := client.Request("GET", templateLoaderPath[templateType]+templateName, "", nil, nil) +func getTemplate(t *testing.T, client ESClient, templateName string) testTemplate { + status, body, err := client.Request("GET", "/_index_template/"+templateName, "", nil, nil) require.NoError(t, err) require.Equal(t, status, 200) @@ -350,29 +353,19 @@ func getTemplate(t *testing.T, client ESClient, templateName string, templateTyp require.NoError(t, err) require.NotNil(t, response) - if templateType == IndexTemplateComponent { - var tmpl map[string]interface{} - components := response["component_templates"].([]interface{}) - for _, ct := range components { - componentTemplate := ct.(map[string]interface{})["component_template"].(map[string]interface{}) - tmpl = componentTemplate["template"].(map[string]interface{}) - } - return testTemplate{ - t: t, - client: client, - MapStr: common.MapStr(tmpl), - } - } + templates, _ := response.GetValue("index_templates") + templatesList, _ := templates.([]interface{}) + templateElem := templatesList[0].(map[string]interface{}) return testTemplate{ t: t, client: client, - MapStr: common.MapStr(response[templateName].(map[string]interface{})), + MapStr: common.MapStr(templateElem["index_template"].(map[string]interface{})), } } func (tt *testTemplate) SourceEnabled() bool { - key := fmt.Sprintf("mappings._source.enabled") + key := fmt.Sprintf("template.mappings._source.enabled") // _source.enabled is true if it's missing (default) b, _ := tt.HasKey(key) @@ -390,7 +383,7 @@ func (tt *testTemplate) SourceEnabled() bool { } func (tt *testTemplate) NumberOfShards() int { - val, err := tt.GetValue("settings.index.number_of_shards") + val, err := tt.GetValue("template.settings.index.number_of_shards") require.NoError(tt.t, err) i, err := strconv.Atoi(val.(string)) diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go index 017f53639fa..429cf7382ab 100644 --- a/libbeat/template/load_test.go +++ b/libbeat/template/load_test.go @@ -32,8 +32,7 @@ import ( func TestFileLoader_Load(t *testing.T) { ver := "7.0.0" prefix := "mock" - order := 1 - info := beat.Info{Version: ver, IndexPrefix: prefix} + info := beat.Info{Beat: "mock", Version: ver, IndexPrefix: prefix} tmplName := fmt.Sprintf("%s-%s", prefix, ver) for name, test := range map[string]struct { @@ -45,39 +44,47 @@ func TestFileLoader_Load(t *testing.T) { }{ "load minimal config info": { body: common.MapStr{ - "index_patterns": []string{"mock-7.0.0-*"}, - "order": order, - "settings": common.MapStr{"index": nil}, + "index_patterns": []string{"mock-7.0.0"}, + "data_stream": struct{}{}, + "priority": 150, + "template": common.MapStr{ + "settings": common.MapStr{"index": nil}}, }, }, "load minimal config with index settings": { settings: TemplateSettings{Index: common.MapStr{"code": "best_compression"}}, body: common.MapStr{ - "index_patterns": []string{"mock-7.0.0-*"}, - "order": order, - "settings": common.MapStr{"index": common.MapStr{"code": "best_compression"}}, + "index_patterns": []string{"mock-7.0.0"}, + "data_stream": struct{}{}, + "priority": 150, + "template": common.MapStr{ + "settings": common.MapStr{"index": common.MapStr{"code": "best_compression"}}}, }, }, "load minimal config with source settings": { settings: TemplateSettings{Source: common.MapStr{"enabled": false}}, body: common.MapStr{ - "index_patterns": []string{"mock-7.0.0-*"}, - "order": order, - "settings": common.MapStr{"index": nil}, - "mappings": common.MapStr{ - "_source": common.MapStr{"enabled": false}, - "_meta": common.MapStr{"beat": prefix, "version": ver}, - "date_detection": false, - "dynamic_templates": nil, - "properties": nil, - }, + "index_patterns": []string{"mock-7.0.0"}, + "data_stream": struct{}{}, + "priority": 150, + "template": common.MapStr{ + "settings": common.MapStr{"index": nil}, + "mappings": common.MapStr{ + "_source": common.MapStr{"enabled": false}, + "_meta": common.MapStr{"beat": prefix, "version": ver}, + "date_detection": false, + "dynamic_templates": nil, + "properties": nil, + }}, }, }, "load config and in-line analyzer fields": { body: common.MapStr{ - "index_patterns": []string{"mock-7.0.0-*"}, - "order": order, - "settings": common.MapStr{"index": nil}, + "index_patterns": []string{"mock-7.0.0"}, + "data_stream": struct{}{}, + "priority": 150, + "template": common.MapStr{ + "settings": common.MapStr{"index": nil}}, }, fields: []byte(`- key: test title: Test fields.yml with analyzer @@ -103,65 +110,66 @@ func TestFileLoader_Load(t *testing.T) { analyzer: simple `), want: common.MapStr{ - "index_patterns": []string{ - "mock-7.0.0-*", - }, - "order": 1, - "mappings": common.MapStr{ - "_meta": common.MapStr{ - "version": "7.0.0", - "beat": "mock", - }, - "date_detection": false, - "dynamic_templates": []common.MapStr{ - { - "strings_as_keyword": common.MapStr{ - "mapping": common.MapStr{ - "ignore_above": 1024, - "type": "keyword", + "index_patterns": []string{"mock-7.0.0"}, + "data_stream": struct{}{}, + "priority": 150, + "template": common.MapStr{ + "mappings": common.MapStr{ + "_meta": common.MapStr{ + "version": "7.0.0", + "beat": "mock", + }, + "date_detection": false, + "dynamic_templates": []common.MapStr{ + { + "strings_as_keyword": common.MapStr{ + "mapping": common.MapStr{ + "ignore_above": 1024, + "type": "keyword", + }, + "match_mapping_type": "string", }, - "match_mapping_type": "string", }, }, - }, - "properties": common.MapStr{ - "code_block_text": common.MapStr{ - "type": "text", - "norms": false, - "analyzer": "test_powershell", - }, - "script_block_text": common.MapStr{ - "type": "text", - "norms": false, - "analyzer": "test_powershell", - }, - "standard_text": common.MapStr{ - "type": "text", - "norms": false, - "analyzer": "simple", + "properties": common.MapStr{ + "code_block_text": common.MapStr{ + "type": "text", + "norms": false, + "analyzer": "test_powershell", + }, + "script_block_text": common.MapStr{ + "type": "text", + "norms": false, + "analyzer": "test_powershell", + }, + "standard_text": common.MapStr{ + "type": "text", + "norms": false, + "analyzer": "simple", + }, }, }, - }, - "settings": common.MapStr{ - "index": common.MapStr{ - "refresh_interval": "5s", - "mapping": common.MapStr{ - "total_fields": common.MapStr{ - "limit": 10000, + "settings": common.MapStr{ + "index": common.MapStr{ + "refresh_interval": "5s", + "mapping": common.MapStr{ + "total_fields": common.MapStr{ + "limit": 10000, + }, }, - }, - "query": common.MapStr{ - "default_field": []string{ - "fields.*", + "query": common.MapStr{ + "default_field": []string{ + "fields.*", + }, }, + "max_docvalue_fields_search": 200, }, - "max_docvalue_fields_search": 200, - }, - "analysis": common.MapStr{ - "analyzer": common.MapStr{ - "test_powershell": map[string]interface{}{ - "type": "pattern", - "pattern": "[\\W&&[^-]]+", + "analysis": common.MapStr{ + "analyzer": common.MapStr{ + "test_powershell": map[string]interface{}{ + "type": "pattern", + "pattern": "[\\W&&[^-]]+", + }, }, }, }, @@ -170,8 +178,7 @@ func TestFileLoader_Load(t *testing.T) { }, "load config and in-line analyzer fields with name collision": { body: common.MapStr{ - "index_patterns": []string{"mock-7.0.0-*"}, - "order": order, + "index_patterns": []string{"mock-7.0.0"}, "settings": common.MapStr{"index": nil}, }, fields: []byte(`- key: test @@ -205,7 +212,7 @@ func TestFileLoader_Load(t *testing.T) { require.NoError(t, err) fl := NewFileLoader(fc) - cfg := DefaultConfig() + cfg := DefaultConfig(info) cfg.Settings = test.settings err = fl.Load(cfg, info, test.fields, false) diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index a9489a35a11..cb13a95aa2d 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -31,7 +31,6 @@ import ( const DefaultField = false var ( - minVersionAlias = common.MustNewVersion("6.4.0") minVersionFieldMeta = common.MustNewVersion("7.6.0") minVersionHistogram = common.MustNewVersion("7.6.0") minVersionWildcard = common.MustNewVersion("7.9.0") @@ -204,11 +203,6 @@ func (p *Processor) scaledFloat(f *mapping.Field, params ...common.MapStr) commo property := p.getDefaultProperties(f) property["type"] = "scaled_float" - if p.EsVersion.IsMajor(2) { - property["type"] = "float" - return property - } - // Set scaling factor scalingFactor := defaultScalingFactor if f.ScalingFactor != 0 && len(f.ObjectTypeParams) == 0 { @@ -270,22 +264,12 @@ func (p *Processor) halfFloat(f *mapping.Field) common.MapStr { property := p.getDefaultProperties(f) property["type"] = "half_float" - if p.EsVersion.IsMajor(2) { - property["type"] = "float" - } return property } func (p *Processor) ip(f *mapping.Field) common.MapStr { property := p.getDefaultProperties(f) - property["type"] = "ip" - - if p.EsVersion.IsMajor(2) { - property["type"] = "string" - property["ignore_above"] = 1024 - property["index"] = "not_analyzed" - } return property } @@ -319,11 +303,6 @@ func (p *Processor) keyword(f *mapping.Field, analyzers common.MapStr) common.Ma property["ignore_above"] = f.IgnoreAbove } - if p.EsVersion.IsMajor(2) { - property["type"] = "string" - property["index"] = "not_analyzed" - } - if len(f.MultiFields) > 0 { fields := common.MapStr{} p.Process(f.MultiFields, stateFromField(f), fields, analyzers) @@ -360,18 +339,8 @@ func (p *Processor) text(f *mapping.Field, analyzers common.MapStr) (properties properties["type"] = "text" - if p.EsVersion.IsMajor(2) { - properties["type"] = "string" - properties["index"] = "analyzed" - if !f.Norms { - properties["norms"] = common.MapStr{ - "enabled": false, - } - } - } else { - if !f.Norms { - properties["norms"] = false - } + if !f.Norms { + properties["norms"] = false } if f.Analyzer.Name != "" { @@ -426,11 +395,6 @@ func (p *Processor) array(f *mapping.Field) common.MapStr { } func (p *Processor) alias(f *mapping.Field) common.MapStr { - // Aliases were introduced in Elasticsearch 6.4, ignore if unsupported - if p.EsVersion.LessThan(minVersionAlias) { - return nil - } - // In case migration is disabled and it's a migration alias, field is not created if !p.Migration && f.MigrationAlias { return nil @@ -483,11 +447,6 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { matchingType = matchType("*", otp.ObjectTypeMappingType) case "text": dynProperties["type"] = "text" - - if p.EsVersion.IsMajor(2) { - dynProperties["type"] = "string" - dynProperties["index"] = "analyzed" - } matchingType = matchType("string", otp.ObjectTypeMappingType) case "keyword": dynProperties["type"] = otp.ObjectType diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index c686f2e4738..fefeb9b4313 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -33,9 +33,6 @@ func TestProcessor(t *testing.T) { trueVar := true p := &Processor{EsVersion: *common.MustNewVersion("7.0.0")} migrationP := &Processor{EsVersion: *common.MustNewVersion("7.0.0"), Migration: true} - pEsVersion2 := &Processor{EsVersion: *common.MustNewVersion("2.0.0")} - pEsVersion64 := &Processor{EsVersion: *common.MustNewVersion("6.4.0")} - pEsVersion63 := &Processor{EsVersion: *common.MustNewVersion("6.3.6")} pEsVersion76 := &Processor{EsVersion: *common.MustNewVersion("7.6.0")} tests := []struct { @@ -81,10 +78,6 @@ func TestProcessor(t *testing.T) { "scaling_factor": 10, }, }, - { - output: pEsVersion2.scaledFloat(&mapping.Field{Type: "scaled_float"}), - expected: common.MapStr{"type": "float"}, - }, { output: p.object(&mapping.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ @@ -111,15 +104,6 @@ func TestProcessor(t *testing.T) { output: p.array(&mapping.Field{Type: "array", Index: &falseVar, ObjectType: "keyword"}), expected: common.MapStr{"index": false, "type": "keyword"}, }, - { - output: pEsVersion64.alias(&mapping.Field{Type: "alias", AliasPath: "a.b"}), - expected: common.MapStr{"path": "a.b", "type": "alias"}, - }, - { - // alias unsupported in ES < 6.4 - output: pEsVersion63.alias(&mapping.Field{Type: "alias", AliasPath: "a.b"}), - expected: nil, - }, { output: p.object(&mapping.Field{Type: "object", Enabled: &falseVar}), expected: common.MapStr{ diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 291d128d0f4..af032487303 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -51,7 +51,6 @@ type Template struct { esVersion common.Version config TemplateConfig migration bool - templateType IndexTemplateType order int priority int } @@ -71,13 +70,17 @@ func New( } name := config.Name + if config.JSON.Enabled { + name = config.JSON.Name + } + if name == "" { name = fmt.Sprintf("%s-%s", beatName, bV.String()) } pattern := config.Pattern if pattern == "" { - pattern = name + "-*" + pattern = name + "*" } event := &beat.Event{ @@ -132,8 +135,6 @@ func New( beatName: beatName, config: config, migration: migration, - templateType: config.Type, - order: config.Order, priority: config.Priority, }, nil } @@ -188,56 +189,25 @@ func (t *Template) LoadBytes(data []byte) (common.MapStr, error) { } // LoadMinimal loads the template only with the given configuration -func (t *Template) LoadMinimal() (common.MapStr, error) { - var m common.MapStr - switch t.templateType { - case IndexTemplateLegacy: - m = t.loadMinimalLegacy() - case IndexTemplateComponent: - m = t.loadMinimalComponent() - case IndexTemplateIndex: - m = t.loadMinimalIndex() - default: - return nil, fmt.Errorf("unknown template type %v", t.templateType) - } - +func (t *Template) LoadMinimal() common.MapStr { + templ := common.MapStr{} if t.config.Settings.Source != nil { - m["mappings"] = buildMappings( - t.beatVersion, t.esVersion, t.beatName, + templ["mappings"] = buildMappings( + t.beatVersion, t.beatName, nil, nil, common.MapStr(t.config.Settings.Source)) } - - return m, nil -} - -func (t *Template) loadMinimalLegacy() common.MapStr { - keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) - return common.MapStr{ - keyPattern: patterns, - "order": t.order, - "settings": common.MapStr{ - "index": t.config.Settings.Index, - }, + templ["settings"] = common.MapStr{ + "index": t.config.Settings.Index, } -} - -func (t *Template) loadMinimalComponent() common.MapStr { return common.MapStr{ - "template": common.MapStr{ - "settings": common.MapStr{ - "index": t.config.Settings.Index, - }, - }, + "template": templ, + "data_stream": struct{}{}, + "priority": t.priority, + "index_patterns": []string{t.GetPattern()}, } } -func (t *Template) loadMinimalIndex() common.MapStr { - m := t.loadMinimalComponent() - m["priority"] = t.priority - return m -} - // GetName returns the name of the template func (t *Template) GetName() string { return t.name @@ -251,45 +221,19 @@ func (t *Template) GetPattern() string { // Generate generates the full template // The default values are taken from the default variable. func (t *Template) Generate(properties, analyzers common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { - switch t.templateType { - case IndexTemplateLegacy: - return t.generateLegacy(properties, analyzers, dynamicTemplates) - case IndexTemplateComponent: - return t.generateComponent(properties, analyzers, dynamicTemplates) - case IndexTemplateIndex: - return t.generateIndex(properties, analyzers, dynamicTemplates) - } - return nil -} + tmpl := t.generateComponent(properties, analyzers, dynamicTemplates) + tmpl["data_stream"] = struct{}{} + tmpl["priority"] = t.priority + tmpl["index_patterns"] = []string{t.GetPattern()} + return tmpl -func (t *Template) generateLegacy(properties, analyzers common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { - keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) - m := common.MapStr{ - keyPattern: patterns, - "order": t.order, - "mappings": buildMappings( - t.beatVersion, t.esVersion, t.beatName, - properties, - append(dynamicTemplates, buildDynTmpl(t.esVersion)), - common.MapStr(t.config.Settings.Source)), - "settings": common.MapStr{ - "index": buildIdxSettings( - t.esVersion, - t.config.Settings.Index, - ), - }, - } - if len(analyzers) != 0 { - m.Put("settings.analysis.analyzer", analyzers) - } - return m } func (t *Template) generateComponent(properties, analyzers common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { m := common.MapStr{ "template": common.MapStr{ "mappings": buildMappings( - t.beatVersion, t.esVersion, t.beatName, + t.beatVersion, t.beatName, properties, append(dynamicTemplates, buildDynTmpl(t.esVersion)), common.MapStr(t.config.Settings.Source)), @@ -302,28 +246,13 @@ func (t *Template) generateComponent(properties, analyzers common.MapStr, dynami }, } if len(analyzers) != 0 { - m.Put("settings.analysis.analyzer", analyzers) + m.Put("template.settings.analysis.analyzer", analyzers) } return m } -func (t *Template) generateIndex(properties, analyzers common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { - tmpl := t.generateComponent(properties, analyzers, dynamicTemplates) - tmpl["priority"] = t.priority - keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) - tmpl[keyPattern] = patterns - return tmpl -} - -func buildPatternSettings(ver common.Version, pattern string) (string, interface{}) { - if ver.Major < 6 { - return "template", pattern - } - return "index_patterns", []string{pattern} -} - func buildMappings( - beatVersion, esVersion common.Version, + beatVersion common.Version, beatName string, properties common.MapStr, dynTmpls []common.MapStr, @@ -343,41 +272,16 @@ func buildMappings( mapping["_source"] = source } - major := esVersion.Major - switch { - case major == 2: - mapping.Put("_all.norms.enabled", false) - mapping = common.MapStr{ - "_default_": mapping, - } - case major < 6: - mapping = common.MapStr{ - "_default_": mapping, - } - case major == 6: - mapping = common.MapStr{ - "doc": mapping, - } - case major >= 7: - // keep typeless structure - } - return mapping } func buildDynTmpl(ver common.Version) common.MapStr { - strMapping := common.MapStr{ - "ignore_above": 1024, - "type": "keyword", - } - if ver.Major == 2 { - strMapping["type"] = "string" - strMapping["index"] = "not_analyzed" - } - return common.MapStr{ "strings_as_keyword": common.MapStr{ - "mapping": strMapping, + "mapping": common.MapStr{ + "ignore_above": 1024, + "type": "keyword", + }, "match_mapping_type": "string", }, } @@ -393,25 +297,14 @@ func buildIdxSettings(ver common.Version, userSettings common.MapStr) common.Map }, } - // number_of_routing shards is only supported for ES version >= 6.1 - // If ES >= 7.0 we can exclude this setting as well. - version61, _ := common.NewVersion("6.1.0") - if !ver.LessThan(version61) && ver.Major < 7 { - indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards) - } - - if ver.Major >= 7 { - // copy defaultFields, as defaultFields is shared global slice. - fields := make([]string, len(defaultFields)) - copy(fields, defaultFields) - fields = append(fields, "fields.*") + // copy defaultFields, as defaultFields is shared global slice. + fields := make([]string, len(defaultFields)) + copy(fields, defaultFields) + fields = append(fields, "fields.*") - indexSettings.Put("query.default_field", fields) - } + indexSettings.Put("query.default_field", fields) - if ver.Major >= 6 { - indexSettings.Put("max_docvalue_fields_search", defaultMaxDocvalueFieldsSearch) - } + indexSettings.Put("max_docvalue_fields_search", defaultMaxDocvalueFieldsSearch) indexSettings.DeepUpdate(userSettings) return indexSettings diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go index 27223ecf6bd..e2430e25ede 100644 --- a/libbeat/template/template_test.go +++ b/libbeat/template/template_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/version" ) @@ -39,30 +40,13 @@ type testTemplate struct { func TestNumberOfRoutingShards(t *testing.T) { const notPresent = 0 // setting missing indicator const settingKey = "number_of_routing_shards" - const fullKey = "settings.index." + settingKey + const fullKey = "template.settings.index." + settingKey cases := map[string]struct { esVersion string set int want int }{ - "Do not set default for older version than 6.1": { - esVersion: "6.0.0", - want: notPresent, - }, - "Default present if ES 6.1.0 is used": { - esVersion: "6.1.0", - want: 30, - }, - "Default present for newer ES 6.x version": { - esVersion: "6.8.2", - want: 30, - }, - "Can overwrite default for ES 6.x": { - esVersion: "6.1.0", - set: 1024, - want: 1024, - }, "Do not set by default for ES 7.x": { esVersion: "7.0.0", want: notPresent, @@ -108,29 +92,20 @@ func TestNumberOfRoutingShards(t *testing.T) { func TestTemplate(t *testing.T) { currentVersion := getVersion("") - - t.Run("for ES 6.x", func(t *testing.T) { - template := createTestTemplate(t, currentVersion, "6.4.0", DefaultConfig()) - template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) - template.Assert("order", 1) - template.Assert("mappings.doc._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) - template.Assert("settings.index.max_docvalue_fields_search", 200) - }) + info := beat.Info{Beat: "testbeat", Version: currentVersion} t.Run("for ES 7.x", func(t *testing.T) { - template := createTestTemplate(t, currentVersion, "7.2.0", DefaultConfig()) - template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) - template.Assert("order", 1) - template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) - template.Assert("settings.index.max_docvalue_fields_search", 200) + template := createTestTemplate(t, currentVersion, "7.10.0", DefaultConfig(info)) + template.Assert("index_patterns", []string{"testbeat-" + currentVersion}) + template.Assert("template.mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + template.Assert("template.settings.index.max_docvalue_fields_search", 200) }) t.Run("for ES 8.x", func(t *testing.T) { - template := createTestTemplate(t, currentVersion, "8.0.0", DefaultConfig()) - template.Assert("index_patterns", []string{"testbeat-" + currentVersion + "-*"}) - template.Assert("order", 1) - template.Assert("mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) - template.Assert("settings.index.max_docvalue_fields_search", 200) + template := createTestTemplate(t, currentVersion, "8.0.0", DefaultConfig(info)) + template.Assert("index_patterns", []string{"testbeat-" + currentVersion}) + template.Assert("template.mappings._meta", common.MapStr{"beat": "testbeat", "version": currentVersion}) + template.Assert("template.settings.index.max_docvalue_fields_search", 200) }) } diff --git a/libbeat/template/testdata/fields.json b/libbeat/template/testdata/fields.json index d95b7a7dabe..01894247b8e 100644 --- a/libbeat/template/testdata/fields.json +++ b/libbeat/template/testdata/fields.json @@ -1,15 +1,17 @@ { "index_patterns": ["foo"], - "settings": { - "number_of_shards": 1 - }, - "mappings": { - "_source": { - "enabled": false + "template": { + "settings": { + "number_of_shards": 1 }, - "properties": { - "host_name": { - "type": "keyword" + "mappings": { + "_source": { + "enabled": false + }, + "properties": { + "host_name": { + "type": "keyword" + } } } } diff --git a/libbeat/template/testdata/fields.yml b/libbeat/template/testdata/fields.yml index afceaeaae69..fcfe105b5e5 100644 --- a/libbeat/template/testdata/fields.yml +++ b/libbeat/template/testdata/fields.yml @@ -3,6 +3,8 @@ description: > Contains all types for testing fields: + - name: '@timestamp' + type: date - name: object type: object diff --git a/libbeat/tests/files/template.json b/libbeat/tests/files/template.json index 655613e32d0..6ebbcfe211d 100644 --- a/libbeat/tests/files/template.json +++ b/libbeat/tests/files/template.json @@ -1,19 +1,21 @@ { "index_patterns": ["bla"], - "settings": { - "number_of_shards": 1 - }, - "mappings": { - "_source": { - "enabled": false + "template": { + "settings": { + "number_of_shards": 1 }, - "properties": { - "host_name": { - "type": "keyword" + "mappings": { + "_source": { + "enabled": false }, - "created_at": { - "type": "date", - "format": "EEE MMM dd HH:mm:ss Z YYYY" + "properties": { + "host_name": { + "type": "keyword" + }, + "created_at": { + "type": "date", + "format": "EEE MMM dd HH:mm:ss Z YYYY" + } } } } diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py index f5bccf2968d..a5caff92c9a 100644 --- a/libbeat/tests/system/beat/common_tests.py +++ b/libbeat/tests/system/beat/common_tests.py @@ -58,7 +58,12 @@ def test_export_template(self): """ output = self.run_export_cmd("template") js = json.loads(output) - assert "index_patterns" in js and "mappings" in js + assert "index_patterns" in js + assert "template" in js + assert "priority" in js + assert "order" not in js + assert "mappings" in js["template"] + assert "settings" in js["template"] def test_export_index_pattern(self): """ diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index edfc178eea8..e6d62a5eae6 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -31,9 +31,6 @@ setup.ilm: {% if ilm.pattern %} pattern: {{ ilm.pattern }} {% endif %} - {% if ilm.rollover_alias %} - rollover_alias: {{ ilm.rollover_alias }} - {% endif %} {% endif %} #================================ Processors ===================================== diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index f789b919bc2..6da6a077c71 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -9,35 +9,45 @@ class IdxMgmt(unittest.TestCase): def __init__(self, client, index): self._client = client self._index = index if index != '' and index != '*' else 'mockbeat' + self.patterns = [self.default_pattern(), "1", datetime.datetime.now().strftime("%Y.%m.%d")] def needs_init(self, s): return s == '' or s == '*' - def delete(self, indices=[], policies=[]): - indices = list([x for x in indices if x != '']) - if not indices: - indices == [self._index] + def delete(self, indices=[], policies=[], data_streams=[]): + for ds in data_streams: + self.delete_data_stream(ds) + self.delete_template(template=ds) for i in indices: self.delete_index_and_alias(i) self.delete_template(template=i) for i in [x for x in policies if x != '']: self.delete_policy(i) + def delete_data_stream(self, data_stream): + try: + resp = self._client.transport.perform_request('DELETE', '/_data_stream/' + data_stream) + except NotFoundError: + pass + def delete_index_and_alias(self, index=""): if self.needs_init(index): index = self._index - try: - self._client.transport.perform_request('DELETE', "/" + index + "*") - except NotFoundError: - pass + for pattern in self.patterns: + index_with_pattern = index+"-"+pattern + try: + self._client.indices.delete(index_with_pattern) + self._client.indices.delete_alias(index, index_with_pattern) + except NotFoundError: + continue def delete_template(self, template=""): if self.needs_init(template): template = self._index try: - self._client.transport.perform_request('DELETE', "/_template/" + template + "*") + self._client.transport.perform_request('DELETE', "/_index_template/" + template) except NotFoundError: pass @@ -54,12 +64,7 @@ def delete_policy(self, policy): def assert_index_template_not_loaded(self, template): with pytest.raises(NotFoundError): - self._client.transport.perform_request('GET', '/_template/' + template) - - def assert_legacy_index_template_loaded(self, template): - resp = self._client.transport.perform_request('GET', '/_template/' + template) - assert template in resp - assert "lifecycle" not in resp[template]["settings"]["index"] + self._client.transport.perform_request('GET', '/_index_template/' + template) def assert_index_template_loaded(self, template): resp = self._client.transport.perform_request('GET', '/_index_template/' + template) @@ -69,40 +74,20 @@ def assert_index_template_loaded(self, template): found = True assert found - def assert_component_template_loaded(self, template): - resp = self._client.transport.perform_request('GET', '/_component_template/' + template) - found = False - print(resp) - for index_template in resp['component_templates']: + def assert_data_stream_created(self, data_stream): + try: + resp = self._client.transport.perform_request('GET', '/_data_stream/' + data_stream) + except NotFoundError: + assert False + + def assert_index_template_index_pattern(self, template, index_pattern): + resp = self._client.transport.perform_request('GET', '/_index_template/' + template) + for index_template in resp['index_templates']: if index_template['name'] == template: + assert index_pattern == index_template['index_template']['index_patterns'] found = True assert found - def assert_ilm_template_loaded(self, template, policy, alias): - resp = self._client.transport.perform_request('GET', '/_template/' + template) - assert resp[template]["settings"]["index"]["lifecycle"]["name"] == policy - assert resp[template]["settings"]["index"]["lifecycle"]["rollover_alias"] == alias - - def assert_index_template_index_pattern(self, template, index_pattern): - resp = self._client.transport.perform_request('GET', '/_template/' + template) - assert template in resp - assert resp[template]["index_patterns"] == index_pattern - - def assert_alias_not_created(self, alias): - resp = self._client.transport.perform_request('GET', '/_alias') - for name, entry in resp.items(): - if alias not in name: - continue - assert entry["aliases"] == {}, entry["aliases"] - - def assert_alias_created(self, alias, pattern=None): - if pattern is None: - pattern = self.default_pattern() - name = alias + "-" + pattern - resp = self._client.transport.perform_request('GET', '/_alias/' + alias) - assert name in resp - assert resp[name]["aliases"][alias]["is_write_index"] == True - def assert_policy_not_created(self, policy): with pytest.raises(NotFoundError): self._client.transport.perform_request('GET', '/_ilm/policy/' + policy) @@ -113,22 +98,14 @@ def assert_policy_created(self, policy): assert resp[policy]["policy"]["phases"]["hot"]["actions"]["rollover"]["max_size"] == "50gb" assert resp[policy]["policy"]["phases"]["hot"]["actions"]["rollover"]["max_age"] == "30d" - def assert_docs_written_to_alias(self, alias, pattern=None): + def assert_docs_written_to_data_stream(self, data_stream): # Refresh the indices to guarantee all documents are available # through the _search API. self._client.transport.perform_request('POST', '/_refresh') - if pattern is None: - pattern = self.default_pattern() - name = alias + "-" + pattern - data = self._client.transport.perform_request('GET', '/' + name + '/_search') + data = self._client.transport.perform_request('GET', '/' + data_stream + '/_search') self.assertGreater(data["hits"]["total"]["value"], 0) def default_pattern(self): d = datetime.datetime.now().strftime("%Y.%m.%d") return d + "-000001" - - def index_for(self, alias, pattern=None): - if pattern is None: - pattern = self.default_pattern() - return "{}-{}".format(alias, pattern) diff --git a/libbeat/tests/system/template/template.go b/libbeat/tests/system/template/template.go index ed284a6ff79..7467f5b60fb 100644 --- a/libbeat/tests/system/template/template.go +++ b/libbeat/tests/system/template/template.go @@ -65,10 +65,10 @@ func testTemplateDefaultFieldLength(beatName string, elasticLicensed bool) func( templateMap := tmpl.Generate(fields, nil, nil) - v, _ := templateMap.GetValue("settings.index.query.default_field") + v, _ := templateMap.GetValue("template.settings.index.query.default_field") defaultValue, ok := v.([]string) if !ok { - t.Fatalf("settings.index.query.default_field value has an unexpected type: %T", v) + t.Fatalf("template.settings.index.query.default_field value has an unexpected type: %T", v) } if len(defaultValue) > MaxDefaultFieldLength { diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py index cd6d19eefbe..2abe8828175 100644 --- a/libbeat/tests/system/test_cmd_setup_index_management.py +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -21,28 +21,29 @@ def setUp(self): self.cmd = "--index-management" # auto-derived default settings, if nothing else is set self.policy_name = self.beat_name - self.index_name = self.alias_name = self.beat_name + "-9.9.9" + self.data_stream = self.beat_name + "-9.9.9" - self.custom_alias = self.beat_name + "_foo" self.custom_policy = self.beat_name + "_bar" self.custom_template = self.beat_name + "_foobar" self.es = self.es_client() - self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], - policies=[self.policy_name, self.custom_policy]) + self.idxmgmt = IdxMgmt(self.es, self.data_stream) + self.idxmgmt.delete(indices=[], + policies=[self.policy_name, self.custom_policy], + data_streams=[self.data_stream]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], - policies=[self.policy_name, self.custom_policy]) + self.idxmgmt.delete(indices=[], + policies=[self.policy_name, self.custom_policy], + data_streams=[self.data_stream]) def render_config(self, **kwargs): self.render_config_template( elasticsearch={"hosts": self.get_elasticsearch_url()}, - es_template_name=self.index_name, + es_template_name=self.data_stream, **kwargs ) @@ -57,45 +58,24 @@ def test_setup_default(self): extra_args=["setup", self.cmd]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) - self.idxmgmt.assert_docs_written_to_alias(self.alias_name) - self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) + self.idxmgmt.assert_index_template_index_pattern(self.data_stream, [self.data_stream]) self.idxmgmt.assert_policy_created(self.policy_name) - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_default(self): - """ - Test setup --index-management with default config - """ - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.cmd]) - - assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) - self.idxmgmt.assert_alias_created(self.alias_name) - self.idxmgmt.assert_policy_created(self.policy_name) - # try deleting policy needs to raise an error as it is in use - with pytest.raises(RequestError): - self.idxmgmt.delete_policy(self.policy_name) - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') def test_setup_template_disabled(self): """ Test setup --index-management when ilm disabled """ + self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + exit_code = self.run_beat(logging_args=["-v", "-e", "-d", "*"], extra_args=["setup", self.cmd, "-E", "setup.template.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_not_loaded(self.index_name) - self.idxmgmt.assert_alias_created(self.index_name) + self.idxmgmt.assert_index_template_not_loaded(self.data_stream+"ba") self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -110,8 +90,7 @@ def test_setup_ilm_disabled(self): "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) - self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -126,7 +105,7 @@ def test_setup_policy_name(self): "-E", "setup.ilm.policy_name=" + self.custom_policy]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_created(self.custom_policy) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -176,58 +155,6 @@ def test_setup_ilm_policy_no_overwrite(self): assert "delete" not in resp[policy_name]["policy"]["phases"] assert "hot" in resp[policy_name]["policy"]["phases"] - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_rollover_alias(self): - """ - Test setup --index-management when ilm.rollover_alias is configured - """ - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.cmd, - "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) - - assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) - self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) - self.idxmgmt.assert_alias_created(self.custom_alias) - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_rollover_alias_with_fieldref(self): - """ - Test setup --index-management when ilm.rollover_alias is configured and using field reference. - """ - aliasFieldRef = "%{[agent.name]}-myalias" - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.cmd, - "-E", "setup.ilm.rollover_alias=" + aliasFieldRef]) - - self.custom_alias = self.beat_name + "-myalias" - - assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) - self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) - self.idxmgmt.assert_alias_created(self.custom_alias) - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_template_name_and_pattern(self): - """ - Test setup --index-management ignores template.name and template.pattern when ilm is enabled - """ - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.cmd, - "-E", "setup.template.name=" + self.custom_template, - "-E", "setup.template.pattern=" + self.custom_template + "*"]) - - assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"]) - self.idxmgmt.assert_alias_created(self.alias_name) - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') def test_setup_template_name_and_pattern_on_ilm_disabled(self): @@ -242,9 +169,8 @@ def test_setup_template_name_and_pattern_on_ilm_disabled(self): "-E", "setup.template.pattern=" + self.custom_template + "*"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.custom_template) + self.idxmgmt.assert_index_template_loaded(self.custom_template) self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"]) - self.idxmgmt.assert_alias_not_created(self.alias_name) self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -261,13 +187,17 @@ def test_setup_template_with_opts(self): "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) # check that settings are overwritten - resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) - assert self.index_name in resp - index = resp[self.index_name]["settings"]["index"] - assert index["number_of_shards"] == "2", index["number_of_shards"] + resp = self.es.transport.perform_request('GET', '/_index_template/' + self.data_stream) + found = False + for index_template in resp["index_templates"]: + if self.data_stream == index_template["name"]: + found = True + index = index_template["index_template"]["template"]["settings"]["index"] + assert index["number_of_shards"] == "2", index["number_of_shards"] + assert found @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -281,22 +211,29 @@ def test_setup_overwrite_template_on_ilm_policy_created(self): exit_code = self.run_beat(logging_args=["-v", "-d", "*"], extra_args=["setup", self.cmd, "-E", "setup.ilm.enabled=false", - "-E", "setup.template.name=" + self.custom_alias, - "-E", "setup.template.pattern=" + self.custom_alias + "*"]) + "-E", "setup.template.priority=160", + "-E", "setup.template.name=" + self.custom_template, + "-E", "setup.template.pattern=" + self.custom_template + "*"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_index_template_loaded(self.custom_template) self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template - exit_code = self.run_beat(extra_args=["setup", self.cmd, - "-E", "setup.template.overwrite=false", - "-E", "setup.template.settings.index.number_of_shards=2", - "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) + exit_code = self.run_beat(extra_args=["setup", "-d", "*", self.cmd, + "-E", "setup.template.overwrite=true", + "-E", "setup.template.name=" + self.custom_template, + "-E", "setup.template.pattern=" + self.custom_template + "*", + "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) + self.idxmgmt.assert_index_template_loaded(self.custom_template) self.idxmgmt.assert_policy_created(self.policy_name) # check that template was overwritten - resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) - assert self.custom_alias in resp - index = resp[self.custom_alias]["settings"]["index"] - assert index["number_of_shards"] == "2", index["number_of_shards"] + resp = self.es.transport.perform_request('GET', '/_index_template/' + self.custom_template) + + found = False + for index_template in resp["index_templates"]: + if index_template["name"] == self.custom_template: + found = True + index = index_template["index_template"]["template"]["settings"]["index"] + assert index["number_of_shards"] == "2", index["number_of_shards"] + assert found diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 672f51327f6..461fc0212f0 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -21,23 +21,24 @@ class TestRunILM(BaseTest): def setUp(self): super(TestRunILM, self).setUp() - self.alias_name = self.index_name = self.beat_name + "-9.9.9" + self.data_stream = self.beat_name + "-9.9.9" self.policy_name = self.beat_name - self.custom_alias = self.beat_name + "_foo" self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() - self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], - policies=[self.policy_name, self.custom_policy]) + self.idxmgmt = IdxMgmt(self.es, self.data_stream) + self.idxmgmt.delete(indices=[], + policies=[self.policy_name, self.custom_policy], + data_streams=[self.data_stream]) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], - policies=[self.policy_name, self.custom_policy]) + self.idxmgmt.delete(indices=[], + policies=[self.policy_name, self.custom_policy], + data_streams=[self.data_stream]) def render_config(self, **kwargs): self.render_config_template( elasticsearch={"hosts": self.get_elasticsearch_url()}, - es_template_name=self.index_name, + es_template_name=self.data_stream, **kwargs ) @@ -45,7 +46,7 @@ def render_config(self, **kwargs): @pytest.mark.tag('integration') def test_ilm_default(self): """ - Test ilm default settings to load ilm policy, write alias and ilm template + Test ilm default settings to load ilm policy, data stream template """ self.render_config() proc = self.start_beat() @@ -54,10 +55,9 @@ def test_ilm_default(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_data_stream_created(self.data_stream) self.idxmgmt.assert_policy_created(self.policy_name) - self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -72,9 +72,9 @@ def test_ilm_disabled(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) - self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_not_created(self.policy_name) + self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -92,71 +92,10 @@ def test_policy_name(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, policy_name, self.alias_name) - self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) + self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream) self.idxmgmt.assert_policy_created(policy_name) - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_rollover_alias(self): - """ - Test settings ilm rollover alias - """ - - self.render_config(ilm={"enabled": True, "rollover_alias": self.custom_alias}) - - proc = self.start_beat() - self.wait_until(lambda: self.log_contains("mockbeat start running.")) - self.wait_until(lambda: self.log_contains(MSG_ILM_POLICY_LOADED)) - self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) - proc.check_kill_and_wait() - - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) - self.idxmgmt.assert_docs_written_to_alias(self.custom_alias) - self.idxmgmt.assert_alias_created(self.custom_alias) - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_pattern(self): - """ - Test setting ilm pattern - """ - - pattern = "1" - self.render_config(ilm={"enabled": True, "pattern": pattern}) - - proc = self.start_beat() - self.wait_until(lambda: self.log_contains("mockbeat start running.")) - self.wait_until(lambda: self.log_contains(MSG_ILM_POLICY_LOADED)) - self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) - proc.check_kill_and_wait() - - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_alias_created(self.alias_name, pattern=pattern) - self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=pattern) - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_pattern_date(self): - """ - Test setting ilm pattern with date - """ - - pattern = "'{now/d}'" - self.render_config(ilm={"enabled": True, "pattern": pattern}) - - proc = self.start_beat() - self.wait_until(lambda: self.log_contains("mockbeat start running.")) - self.wait_until(lambda: self.log_contains(MSG_ILM_POLICY_LOADED)) - self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) - proc.check_kill_and_wait() - - resolved_pattern = datetime.datetime.now().strftime("%Y.%m.%d") - - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_alias_created(self.alias_name, pattern=resolved_pattern) - self.idxmgmt.assert_docs_written_to_alias(self.alias_name, pattern=resolved_pattern) - class TestCommandSetupILMPolicy(BaseTest): """ @@ -168,26 +107,27 @@ def setUp(self): self.setupCmd = "--index-management" - self.alias_name = self.index_name = self.beat_name + "-9.9.9" + self.data_stream = self.beat_name + "-9.9.9" self.policy_name = self.beat_name - self.custom_alias = self.beat_name + "_foo" self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() - self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], - policies=[self.policy_name, self.custom_policy]) + self.idxmgmt = IdxMgmt(self.es, self.data_stream) + self.idxmgmt.delete(indices=[], + policies=[self.policy_name, self.custom_policy], + data_streams=[self.data_stream]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy], - policies=[self.policy_name, self.custom_policy]) + self.idxmgmt.delete(indices=[], + policies=[self.policy_name, self.custom_policy], + data_streams=[self.data_stream]) def render_config(self, **kwargs): self.render_config_template( elasticsearch={"hosts": self.get_elasticsearch_url()}, - es_template_name=self.index_name, + es_template_name=self.data_stream, **kwargs ) @@ -203,9 +143,7 @@ def test_setup_ilm_default(self): extra_args=["setup", self.setupCmd]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) - self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"]) - self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.data_stream, [self.data_stream]) self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -221,8 +159,7 @@ def test_setup_ilm_disabled(self): "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) - self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_not_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -238,25 +175,9 @@ def test_policy_name(self): "-E", "setup.ilm.policy_name=" + self.custom_policy]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_created(self.custom_policy) - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_rollover_alias(self): - """ - Test ilm policy setup when rollover_alias is configured - """ - self.render_config() - - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.setupCmd, - "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) - - assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) - self.idxmgmt.assert_alias_created(self.custom_alias) - class TestCommandExportILMPolicy(BaseTest): """ @@ -277,9 +198,6 @@ def assert_log_contains_policy(self): assert self.log_contains('"max_age": "30d"') assert self.log_contains('"max_size": "50gb"') - def assert_log_contains_write_alias(self): - assert self.log_contains(re.compile('Index Alias .* successfully created.')) - def test_default(self): """ Test ilm-policy export with default config @@ -290,7 +208,6 @@ def test_default(self): assert exit_code == 0 self.assert_log_contains_policy() - self.assert_log_contains_write_alias() def test_load_disabled(self): """ @@ -302,7 +219,6 @@ def test_load_disabled(self): assert exit_code == 0 self.assert_log_contains_policy() - self.assert_log_contains_write_alias() def test_changed_policy_name(self): """ @@ -315,7 +231,6 @@ def test_changed_policy_name(self): assert exit_code == 0 self.assert_log_contains_policy() - self.assert_log_contains_write_alias() def test_export_to_file_absolute_path(self): """ diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 6a81a6bd036..abed68332a8 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -110,7 +110,7 @@ def test_json_template(self): self.wait_until(lambda: self.log_contains('Template with name \\\"bla\\\" loaded.')) proc.check_kill_and_wait() - result = es.transport.perform_request('GET', '/_template/' + template_name) + result = es.transport.perform_request('GET', '/_index_template/' + template_name) assert len(result) == 1 def get_host(self): @@ -125,14 +125,14 @@ class TestRunTemplate(BaseTest): def setUp(self): super(TestRunTemplate, self).setUp() # auto-derived default settings, if nothing else is set - self.index_name = self.beat_name + "-9.9.9" + self.data_stream = self.beat_name + "-9.9.9" self.es = self.es_client() - self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.index_name]) + self.idxmgmt = IdxMgmt(self.es, self.data_stream) + self.idxmgmt.delete(data_streams=[self.data_stream]) def tearDown(self): - self.idxmgmt.delete(indices=[self.index_name]) + self.idxmgmt.delete(data_streams=[self.data_stream]) def render_config(self, **kwargs): self.render_config_template( @@ -153,9 +153,8 @@ def test_template_default(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.beat_name, self.index_name) - self.idxmgmt.assert_alias_created(self.index_name) - self.idxmgmt.assert_docs_written_to_alias(self.index_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) + self.idxmgmt.assert_docs_written_to_data_stream(self.data_stream) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -169,7 +168,7 @@ def test_template_disabled(self): self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) proc.check_kill_and_wait() - self.idxmgmt.assert_index_template_not_loaded(self.index_name) + self.idxmgmt.assert_index_template_not_loaded(self.data_stream) class TestCommandSetupTemplate(BaseTest): @@ -182,18 +181,17 @@ def setUp(self): # auto-derived default settings, if nothing else is set self.setupCmd = "--index-management" - self.index_name = self.beat_name + "-9.9.9" - self.custom_alias = self.beat_name + "_foo" + self.data_stream = self.beat_name + "-9.9.9" self.policy_name = self.beat_name self.es = self.es_client() - self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name], policies=[self.policy_name]) + self.idxmgmt = IdxMgmt(self.es, self.data_stream) + self.idxmgmt.delete(indices=[self.data_stream], policies=[self.policy_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(indices=[self.custom_alias, self.index_name], policies=[self.policy_name]) + self.idxmgmt.delete(indices=[self.data_stream], policies=[self.policy_name]) def render_config(self, **kwargs): self.render_config_template( @@ -212,8 +210,7 @@ def test_setup(self): extra_args=["setup", self.setupCmd]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.policy_name, self.index_name) - self.idxmgmt.assert_alias_created(self.index_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -227,10 +224,9 @@ def test_setup_template_default(self): extra_args=["setup", self.setupCmd]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.index_name, self.policy_name, self.index_name) - self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) + self.idxmgmt.assert_index_template_loaded(self.data_stream) + self.idxmgmt.assert_index_template_index_pattern(self.data_stream, [self.data_stream]) - self.idxmgmt.assert_alias_created(self.index_name) self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -245,9 +241,8 @@ def test_setup_template_disabled(self): "-E", "setup.template.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_index_template_not_loaded(self.index_name) + self.idxmgmt.assert_index_template_not_loaded(self.data_stream) - self.idxmgmt.assert_alias_created(self.index_name) self.idxmgmt.assert_policy_created(self.policy_name) @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @@ -263,28 +258,17 @@ def test_setup_template_with_opts(self): "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.index_name) + self.idxmgmt.assert_index_template_loaded(self.data_stream) # check that settings are overwritten - resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) - assert self.index_name in resp - index = resp[self.index_name]["settings"]["index"] - assert index["number_of_shards"] == "2", index["number_of_shards"] - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_template_with_ilm_changed_pattern(self): - """ - Test template setup with changed ilm.rollover_alias config - """ - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.setupCmd, - "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) - - assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) - self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) + resp = self.es.transport.perform_request('GET', '/_index_template/' + self.data_stream) + found = False + for index_template in resp["index_templates"]: + if self.data_stream == index_template["name"]: + found = True + index = index_template["index_template"]["template"]["settings"]["index"] + assert index["number_of_shards"] == "2", index["number_of_shards"] + assert found @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -293,58 +277,20 @@ def test_template_created_on_ilm_policy_created(self): Test template setup overwrites template when new ilm policy is created """ - # ensure template with ilm rollover_alias name is created, but ilm policy not yet self.render_config() exit_code = self.run_beat(logging_args=["-v", "-d", "*"], extra_args=["setup", self.setupCmd, - "-E", "setup.ilm.enabled=false", - "-E", "setup.template.name=" + self.custom_alias, - "-E", "setup.template.pattern=" + self.custom_alias + "*"]) + "-E", "setup.ilm.enabled=false"]) assert exit_code == 0 - self.idxmgmt.assert_legacy_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_index_template_loaded(self.data_stream) self.idxmgmt.assert_policy_not_created(self.policy_name) # ensure ilm policy is created, triggering overwriting existing template exit_code = self.run_beat(extra_args=["setup", self.setupCmd, "-E", "setup.template.overwrite=false", - "-E", "setup.template.settings.index.number_of_shards=2", - "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) + "-E", "setup.template.settings.index.number_of_shards=2"]) assert exit_code == 0 - self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) self.idxmgmt.assert_policy_created(self.policy_name) - # check that template was overwritten - resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) - assert self.custom_alias in resp - index = resp[self.custom_alias]["settings"]["index"] - assert index["number_of_shards"] == "2", index["number_of_shards"] - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_template_index(self): - """ - Test template setup of new index templates - """ - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.setupCmd, - "-E", "setup.template.type=index"]) - - assert exit_code == 0 - self.idxmgmt.assert_index_template_loaded(self.index_name) - - @unittest.skipUnless(INTEGRATION_TESTS, "integration test") - @pytest.mark.tag('integration') - def test_setup_template_component(self): - """ - Test template setup of component index templates - """ - self.render_config() - exit_code = self.run_beat(logging_args=["-v", "-d", "*"], - extra_args=["setup", self.setupCmd, - "-E", "setup.template.type=component"]) - - assert exit_code == 0 - self.idxmgmt.assert_component_template_loaded(self.index_name) class TestCommandExportTemplate(BaseTest): @@ -375,23 +321,7 @@ def test_default(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name + "-*") - - def test_changed_index_pattern(self): - """ - Test export template with changed index pattern - """ - self.render_config_template(self.beat_name, self.output, - fields=self.output) - alias_name = "mockbeat-ilm-index-pattern" - - exit_code = self.run_beat( - extra_args=["export", "template", - "-E", "setup.ilm.rollover_alias=" + alias_name], - config=self.config) - - assert exit_code == 0 - self.assert_log_contains_template(alias_name + "-*") + self.assert_log_contains_template(self.template_name) def test_load_disabled(self): """ @@ -404,7 +334,7 @@ def test_load_disabled(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name + "-*") + self.assert_log_contains_template(self.template_name) def test_export_to_file_absolute_path(self): """ @@ -424,7 +354,7 @@ def test_export_to_file_absolute_path(self): with open(file) as f: template = json.load(f) assert 'index_patterns' in template - assert template['index_patterns'] == [self.template_name + '-*'], template + assert template['index_patterns'] == [self.template_name], template os.remove(file) @@ -447,6 +377,6 @@ def test_export_to_file_relative_path(self): with open(file) as f: template = json.load(f) assert 'index_patterns' in template - assert template['index_patterns'] == [self.template_name + '-*'], template + assert template['index_patterns'] == [self.template_name], template os.remove(file) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index c036073f011..0826671a80e 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1988,19 +1988,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default metricbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "metricbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "metricbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "metricbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "metricbeat-%{[agent.version]}-*" +#setup.template.pattern: "metricbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -2047,17 +2041,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'metricbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'metricbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/metricbeat/tests/system/test_template.py b/metricbeat/tests/system/test_template.py index 75b366ab15d..17df56cf552 100644 --- a/metricbeat/tests/system/test_template.py +++ b/metricbeat/tests/system/test_template.py @@ -41,7 +41,7 @@ def test_export_template(self): break t = json.loads(template_content) - properties = t["mappings"]["properties"] + properties = t["template"]["mappings"]["properties"] # Check libbeat fields assert properties["@timestamp"] == {"type": "date"} diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 5f931ddde70..1a51aeb6b4d 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1640,19 +1640,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default packetbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "packetbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "packetbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "packetbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "packetbeat-%{[agent.version]}-*" +#setup.template.pattern: "packetbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1699,17 +1693,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'packetbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'packetbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 305d1b5acce..17901a0b179 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1074,19 +1074,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default winlogbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "winlogbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "winlogbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "winlogbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "winlogbeat-%{[agent.version]}-*" +#setup.template.pattern: "winlogbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1133,17 +1127,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'winlogbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'winlogbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index f52cc6feab1..a83cc1742e2 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1201,19 +1201,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default auditbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "auditbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "auditbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "auditbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "auditbeat-%{[agent.version]}-*" +#setup.template.pattern: "auditbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1260,17 +1254,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'auditbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'auditbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index fae9d78fb5d..d6694e9e700 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4231,19 +4231,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default filebeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "filebeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "filebeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "filebeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "filebeat-%{[agent.version]}-*" +#setup.template.pattern: "filebeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -4290,17 +4284,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'filebeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'filebeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml index 87c3deacb97..7b30ee7d38a 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml @@ -18,9 +18,9 @@ processors: replacement: ' ' - trim: field: message -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: '@timestamp' + field: event.created - grok: field: message patterns: diff --git a/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json b/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json index 45a57fffd05..78b92a4d5fd 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json +++ b/x-pack/filebeat/module/ibmmq/errorlog/test/AMQERR01_QM1.log-expected.json @@ -1,5 +1,6 @@ [ { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -32,6 +33,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -64,6 +66,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -96,6 +99,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -128,6 +132,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -160,6 +165,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -192,6 +198,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -224,6 +231,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -256,6 +264,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -288,6 +297,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -320,6 +330,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -352,6 +363,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -384,6 +396,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.788Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -416,6 +429,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -448,6 +462,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -480,6 +495,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -512,6 +528,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -544,6 +561,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -576,6 +594,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -608,6 +627,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -640,6 +660,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -672,6 +693,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", @@ -704,6 +726,7 @@ "user.name": "felix" }, { + "@timestamp": "2021-12-03T15:44:37.789Z", "event.dataset": "ibmmq.errorlog", "event.kind": "event", "event.module": "ibmmq", diff --git a/x-pack/filebeat/module/snyk/audit/config/config.yml b/x-pack/filebeat/module/snyk/audit/config/config.yml index 7476d4dd1cf..4932c7133a1 100644 --- a/x-pack/filebeat/module/snyk/audit/config/config.yml +++ b/x-pack/filebeat/module/snyk/audit/config/config.yml @@ -67,13 +67,6 @@ processors: - fingerprint: fields: ["json.orgId", "json.created", "json.event"] target_field: "@metadata._id" - - script: - lang: javascript - id: my_filter - source: > - function process(event) { - event.Put("@metadata.op_type", "index"); - } - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml index a78cc297493..31819689716 100644 --- a/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml +++ b/x-pack/filebeat/module/snyk/vulnerabilities/config/config.yml @@ -90,13 +90,6 @@ processors: - fingerprint: fields: ["json.issue.id"] target_field: "@metadata._id" - - script: - lang: javascript - id: my_filter - source: > - function process(event) { - event.Put("@metadata.op_type", "index"); - } - add_fields: target: '' fields: diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 5ea23a3b0db..e4167320640 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -939,19 +939,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default functionbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "functionbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "functionbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "functionbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "functionbeat-%{[agent.version]}-*" +#setup.template.pattern: "functionbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -998,17 +992,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'functionbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'functionbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 8f0f019626b..c3ae5c01214 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1291,19 +1291,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default heartbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "heartbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "heartbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "heartbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "heartbeat-%{[agent.version]}-*" +#setup.template.pattern: "heartbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1350,17 +1344,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'heartbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'heartbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4d1d41aa0d5..e75eb46dda3 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2509,19 +2509,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default metricbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "metricbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "metricbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "metricbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "metricbeat-%{[agent.version]}-*" +#setup.template.pattern: "metricbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -2568,17 +2562,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'metricbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'metricbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index edb3adcfb1d..a0c021361e9 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -658,19 +658,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default osquerybeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "osquerybeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "osquerybeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "osquerybeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "osquerybeat-%{[agent.version]}-*" +#setup.template.pattern: "osquerybeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -717,17 +711,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'osquerybeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'osquerybeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 5f931ddde70..1a51aeb6b4d 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1640,19 +1640,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default packetbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "packetbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "packetbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "packetbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "packetbeat-%{[agent.version]}-*" +#setup.template.pattern: "packetbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1699,17 +1693,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'packetbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'packetbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'. diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index d6e9d30c50d..cff71187ffb 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1117,19 +1117,13 @@ output.elasticsearch: # Set to false to disable template loading. #setup.template.enabled: true -# Select the kind of index template. From Elasticsearch 7.8, it is possible to -# use component templates. Available options: legacy, component, index. -# By default winlogbeat uses the legacy index templates. -#setup.template.type: legacy - # Template name. By default the template name is "winlogbeat-%{[agent.version]}" # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. #setup.template.name: "winlogbeat-%{[agent.version]}" -# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. -# The first part is the version of the beat and then -* is used to match all daily indices. +# Template pattern. By default the template pattern is "winlogbeat-%{[agent.version]}" to apply to the default index settings. # The template name and pattern has to be set in case the Elasticsearch index pattern is modified. -#setup.template.pattern: "winlogbeat-%{[agent.version]}-*" +#setup.template.pattern: "winlogbeat-%{[agent.version]}" # Path to fields.yml file to generate the template #setup.template.fields: "${path.config}/fields.yml" @@ -1176,17 +1170,8 @@ setup.template.settings: # output.elasticsearch.index is ignored, and the write alias is used to set the # index name. -# Enable ILM support. Valid values are true, false, and auto. When set to auto -# (the default), the Beat uses index lifecycle management when it connects to a -# cluster that supports ILM; otherwise, it creates daily indices. -#setup.ilm.enabled: auto - -# Set the prefix used in the index lifecycle write alias name. The default alias -# name is 'winlogbeat-%{[agent.version]}'. -#setup.ilm.rollover_alias: 'winlogbeat' - -# Set the rollover index pattern. The default is "%{now/d}-000001". -#setup.ilm.pattern: "{now/d}-000001" +# Enable ILM support. Valid values are true, false. +#setup.ilm.enabled: true # Set the lifecycle policy name. The default policy name is # 'beatname'.