-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[UII] Fill in empty values for
constant_keyword fields from existin…
…g mappings (#188145) ## Summary Resolves #178528. Some packages declare `constant_keyword` type fields without an explicit value. This causes ES to fill in the value in the mappings using the first ingested value. When upgrading this type of package & field after the value has already been populated in this way, the mappings update fail due to pushing a `null` value into an existing value, triggering unnecessary rollovers. This PR fixes that by filling in the empty values from the existing mappings. ## Test 1. On an empty cluster, turn on debug logs 2. Set up Fleet Server policy and Fleet Server agent 3. Force install old version of Elastic Agent integration, v1.19.2: ``` POST kbn:/api/fleet/epm/packages/elastic_agent/1.19.2 { "force": true } ``` 4. Create a new empty policy, **deselect system and agent monitoring** (otherwise the integration will be upgraded, we do not want this yet) 5. Manually add Elastic Agent integration v1.19.2 to the new policy 6. Edit the policy to enable logs and metrics monitoring 7. Enroll agent into the policy, confirm that monitoring logs and metrics are being ingested and that a value exists for `event.dataset` mapping for the logs: ``` GET logs-elastic_agent*/_mappings ``` ``` "dataset": { "type": "constant_keyword", "value": "elastic_agent" } ``` 9. Upgrade Elastic Agent integration to v1.20.0 (note we are not upgrading to the newest versions, 2.0+, because these **are** expected to trigger rollovers for some data streams): ``` POST kbn:/api/fleet/epm/packages/elastic_agent/1.20.0 { "force": true } ``` 10. Confirm in Kibana logs that no rollovers triggered during the upgrade 11. Confirm that there is still only 1 backing index for monitoring logs: ``` GET logs-elastic_agent* ``` ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
- Loading branch information
Showing
3 changed files
with
270 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
226 changes: 226 additions & 0 deletions
226
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/utils.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,226 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
| import { fillConstantKeywordValues } from './utils'; | ||
|
|
||
| describe('fillConstantKeywordValues', () => { | ||
| const oldMappings = { | ||
| dynamic: false, | ||
| _meta: { | ||
| managed_by: 'fleet', | ||
| managed: true, | ||
| package: { | ||
| name: 'elastic_agent', | ||
| }, | ||
| }, | ||
| dynamic_templates: [ | ||
| { | ||
| ecs_timestamp: { | ||
| match: '@timestamp', | ||
| mapping: { | ||
| ignore_malformed: false, | ||
| type: 'date', | ||
| }, | ||
| }, | ||
| }, | ||
| ], | ||
| date_detection: false, | ||
| properties: { | ||
| '@timestamp': { | ||
| type: 'date', | ||
| ignore_malformed: false, | ||
| }, | ||
| load: { | ||
| properties: { | ||
| '1': { | ||
| type: 'double', | ||
| }, | ||
| '5': { | ||
| type: 'double', | ||
| }, | ||
| '15': { | ||
| type: 'double', | ||
| }, | ||
| }, | ||
| }, | ||
| event: { | ||
| properties: { | ||
| agent_id_status: { | ||
| type: 'keyword', | ||
| ignore_above: 1024, | ||
| }, | ||
| dataset: { | ||
| type: 'constant_keyword', | ||
| value: 'elastic_agent.metricbeat', | ||
| }, | ||
| ingested: { | ||
| type: 'date', | ||
| format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', | ||
| ignore_malformed: false, | ||
| }, | ||
| }, | ||
| }, | ||
| message: { | ||
| type: 'match_only_text', | ||
| }, | ||
| 'dot.field': { | ||
| type: 'keyword', | ||
| }, | ||
| constant_keyword_without_value: { | ||
| type: 'constant_keyword', | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| const newMappings = { | ||
| dynamic: false, | ||
| _meta: { | ||
| managed_by: 'fleet', | ||
| managed: true, | ||
| package: { | ||
| name: 'elastic_agent', | ||
| }, | ||
| }, | ||
| dynamic_templates: [ | ||
| { | ||
| ecs_timestamp: { | ||
| match: '@timestamp', | ||
| mapping: { | ||
| ignore_malformed: false, | ||
| type: 'date', | ||
| }, | ||
| }, | ||
| }, | ||
| ], | ||
| date_detection: false, | ||
| properties: { | ||
| '@timestamp': { | ||
| type: 'date', | ||
| ignore_malformed: false, | ||
| }, | ||
| load: { | ||
| properties: { | ||
| '1': { | ||
| type: 'double', | ||
| }, | ||
| '5': { | ||
| type: 'double', | ||
| }, | ||
| '15': { | ||
| type: 'double', | ||
| }, | ||
| }, | ||
| }, | ||
| event: { | ||
| properties: { | ||
| agent_id_status: { | ||
| type: 'keyword', | ||
| ignore_above: 1024, | ||
| }, | ||
| dataset: { | ||
| type: 'constant_keyword', | ||
| }, | ||
| ingested: { | ||
| type: 'date', | ||
| format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', | ||
| ignore_malformed: false, | ||
| }, | ||
| }, | ||
| }, | ||
| message: { | ||
| type: 'match_only_text', | ||
| }, | ||
| 'dot.field': { | ||
| type: 'keyword', | ||
| }, | ||
| some_new_field: { | ||
| type: 'keyword', | ||
| }, | ||
| constant_keyword_without_value: { | ||
| type: 'constant_keyword', | ||
| }, | ||
| }, | ||
| }; | ||
|
|
||
| it('should fill in missing constant_keyword values from old mappings correctly', () => { | ||
| // @ts-ignore | ||
| expect(fillConstantKeywordValues(oldMappings, newMappings)).toEqual({ | ||
| dynamic: false, | ||
| _meta: { | ||
| managed_by: 'fleet', | ||
| managed: true, | ||
| package: { | ||
| name: 'elastic_agent', | ||
| }, | ||
| }, | ||
| dynamic_templates: [ | ||
| { | ||
| ecs_timestamp: { | ||
| match: '@timestamp', | ||
| mapping: { | ||
| ignore_malformed: false, | ||
| type: 'date', | ||
| }, | ||
| }, | ||
| }, | ||
| ], | ||
| date_detection: false, | ||
| properties: { | ||
| '@timestamp': { | ||
| type: 'date', | ||
| ignore_malformed: false, | ||
| }, | ||
| load: { | ||
| properties: { | ||
| '1': { | ||
| type: 'double', | ||
| }, | ||
| '5': { | ||
| type: 'double', | ||
| }, | ||
| '15': { | ||
| type: 'double', | ||
| }, | ||
| }, | ||
| }, | ||
| event: { | ||
| properties: { | ||
| agent_id_status: { | ||
| type: 'keyword', | ||
| ignore_above: 1024, | ||
| }, | ||
| dataset: { | ||
| type: 'constant_keyword', | ||
| value: 'elastic_agent.metricbeat', | ||
| }, | ||
| ingested: { | ||
| type: 'date', | ||
| format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', | ||
| ignore_malformed: false, | ||
| }, | ||
| }, | ||
| }, | ||
| message: { | ||
| type: 'match_only_text', | ||
| }, | ||
| 'dot.field': { | ||
| type: 'keyword', | ||
| }, | ||
| some_new_field: { | ||
| type: 'keyword', | ||
| }, | ||
| constant_keyword_without_value: { | ||
| type: 'constant_keyword', | ||
| }, | ||
| }, | ||
| }); | ||
| }); | ||
|
|
||
| it('should return the same mappings if old mappings are not provided', () => { | ||
| // @ts-ignore | ||
| expect(fillConstantKeywordValues({}, newMappings)).toMatchObject(newMappings); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters